server/libfic/file.go

package fic

import (
	"compress/gzip"
	"crypto"
	_ "crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"hash"
	"io"
	"os"
	"path"
	"strings"

	_ "golang.org/x/crypto/blake2b"
)

// FilesDir stores the location where files to be served are stored.
var FilesDir string = "./FILES/"

// OptionalDigest permits to avoid importation failure if no digest are given.
var OptionalDigest bool = false

// StrongDigest forces the use of BLAKE2b hash in place of SHA1 (or mixed SHA1/BLAKE2b).
var StrongDigest bool = false

// Set PlainDigest if digest errors should contain the whole calculated hash, to be paste directly into DIGESTS file.
var PlainDigest bool = false

// EFile represents a challenge file.
type EFile struct {
	Id int64 `json:"id,omitempty"`
	// origin holds the import relative path of the file
	origin string
	// Path is the location where the file is stored, relatively to FilesDir
	Path string `json:"path"`
	// IdExercice is the identifier of the underlying challenge
	IdExercice int64 `json:"idExercice,omitempty"`
	// Name is the title displayed to players
	Name string `json:"name"`
	// Checksum stores the cached hash of the file
	Checksum []byte `json:"checksum"`
	// ChecksumShown stores the hash of the downloaded file (in case of gzipped content)
	ChecksumShown []byte `json:"checksum_shown"`
	// Size contains the cached size of the file
	Size int64 `json:"size"`
	// Disclaimer contains a string to display before downloading the content
	Disclaimer string `json:"disclaimer"`
	// Published indicates if the file should be shown or not
	Published bool `json:"published"`
}

// NewDummyFile creates an EFile, without any link to an actual Exercice File.
// It is used to check the file validity
func (e *Exercice) NewDummyFile(origin string, dest string, checksum []byte, checksumShown []byte, disclaimer string, size int64) *EFile {
	return &EFile{
		Id:            0,
		origin:        origin,
		Path:          dest,
		IdExercice:    e.Id,
		Name:          path.Base(origin),
		Checksum:      checksum,
		ChecksumShown: checksumShown,
		Size:          size,
		Disclaimer:    disclaimer,
		Published:     true,
	}
}

// GetFiles returns a list of all files living in the database.
func GetFiles() ([]*EFile, error) {
	if rows, err := DBQuery("SELECT id_file, id_exercice, origin, path, name, cksum, cksum_shown, size, disclaimer, published FROM exercice_files"); err != nil {
		return nil, err
	} else {
		defer rows.Close()

		files := []*EFile{}
		for rows.Next() {
			f := &EFile{}
			if err := rows.Scan(&f.Id, &f.IdExercice, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size, &f.Disclaimer, &f.Published); err != nil {
				return nil, err
			}
			files = append(files, f)
		}
		if err := rows.Err(); err != nil {
			return nil, err
		}

		return files, nil
	}
}

// GetFile retrieves the file with the given id.
func GetFile(id int64) (f *EFile, err error) {
	f = &EFile{}
	err = DBQueryRow("SELECT id_file, origin, path, name, cksum, cksum_shown, size, disclaimer, published FROM exercice_files WHERE id_file = ?", id).Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size, &f.Disclaimer, &f.Published)
	return
}

func (e *Exercice) GetFile(id int64) (f *EFile, err error) {
	f = &EFile{}
	err = DBQueryRow("SELECT id_file, origin, path, name, cksum, cksum_shown, size, disclaimer, published FROM exercice_files WHERE id_file = ? AND id_exercice = ?", id, e.Id).Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size, &f.Disclaimer, &f.Published)
	return
}

// GetFileByPath retrieves the file that should be found at the given location.
func GetFileByPath(path string) (*EFile, error) {
	path = strings.TrimPrefix(path, FilesDir)

	f := &EFile{}
	if err := DBQueryRow("SELECT id_file, origin, path, id_exercice, name, cksum, cksum_shown, size, disclaimer, published FROM exercice_files WHERE path = ?", path).Scan(&f.Id, &f.origin, &f.Path, &f.IdExercice, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size, &f.Disclaimer, &f.Published); err != nil {
		return f, err
	}

	return f, nil
}

// GetFileByFilename retrieves the file that should be called so.
func (e *Exercice) GetFileByFilename(filename string) (f *EFile, err error) {
	filename = path.Base(filename)

	f = &EFile{}
	err = DBQueryRow("SELECT id_file, origin, path, id_exercice, name, cksum, cksum_shown, size, disclaimer, published FROM exercice_files WHERE id_exercice = ? AND origin LIKE ?", e.Id, "%/"+filename).Scan(&f.Id, &f.origin, &f.Path, &f.IdExercice, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size, &f.Disclaimer, &f.Published)

	return
}

// GetFiles returns a list of files coming with the challenge.
func (e *Exercice) GetFiles() ([]*EFile, error) {
	if rows, err := DBQuery("SELECT id_file, origin, path, name, cksum, cksum_shown, size, disclaimer, published FROM exercice_files WHERE id_exercice = ?", e.Id); err != nil {
		return nil, err
	} else {
		defer rows.Close()

		files := []*EFile{}
		for rows.Next() {
			f := &EFile{}
			f.IdExercice = e.Id
			if err := rows.Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size, &f.Disclaimer, &f.Published); err != nil {
				return nil, err
			}
			files = append(files, f)
		}
		if err := rows.Err(); err != nil {
			return nil, err
		}

		return files, nil
	}
}

// GetFileByPath retrieves the file that should be found at the given location, limited to the challenge files.
func (e *Exercice) GetFileByPath(path string) (*EFile, error) {
	path = strings.TrimPrefix(path, FilesDir)

	f := &EFile{}
	if err := DBQueryRow("SELECT id_file, origin, path, name, cksum, cksum_shown, size, disclaimer, published FROM exercice_files WHERE id_exercice = ? AND path = ?", e.Id, path).Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size, &f.Disclaimer, &f.Published); err != nil {
		return nil, err
	}

	return f, nil
}

// minifyHash returns only the begining and the end of the given hash.
// Use this function to ensure people doesn't fill their file with our calculated hash.
func minifyHash(hash string) string {
	if PlainDigest {
		return hash
	} else {
		return hash[0:len(hash)/3] + "..." + hash[len(hash)/2:]
	}
}

// CheckBufferHash checks if the bufio has the given digest.
func CreateHashBuffers(rd io.Reader) (*hash.Hash, *hash.Hash) {
	hash160 := crypto.SHA1.New()
	hash512 := crypto.BLAKE2b_512.New()

	w := io.MultiWriter(hash160, hash512)

	io.Copy(w, rd)

	return &hash160, &hash512
}

// CheckBufferHash checks if the bufio has the given digest.
func CheckBufferHash(hash160 *hash.Hash, hash512 *hash.Hash, digest []byte) ([]byte, error) {
	result160 := (*hash160).Sum(nil)
	result512 := (*hash512).Sum(nil)

	if len(digest) != len(result512) {
		if len(digest) != len(result160) {
			return []byte{}, errors.New("Digests doesn't match: calculated: sha1:" + minifyHash(hex.EncodeToString(result160)) + " & blake2b:" + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
		} else if StrongDigest {
			return []byte{}, errors.New("Invalid digests: SHA-1 checksums are no more accepted. Calculated sha1:" + minifyHash(hex.EncodeToString(result160)) + " & blake2b:" + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
		}

		for k := range result160 {
			if result160[k] != digest[k] {
				return []byte{}, errors.New("Digests doesn't match: calculated: sha1:" + minifyHash(hex.EncodeToString(result160)) + " & blake2b:" + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
			}
		}
	} else {
		for k := range result512 {
			if result512[k] != digest[k] {
				return []byte{}, errors.New("Digests doesn't match: calculated: " + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
			}
		}
	}

	return result512, nil
}

// checkFileHash checks if the file at the given filePath has the given digest.
// It also returns the file's size.
func checkFileHash(filePath string, digest []byte) (dgst []byte, size int64, err error) {
	if digest == nil {
		return []byte{}, 0, errors.New("no digest given")
	} else if fi, errr := os.Stat(filePath); errr != nil {
		return []byte{}, 0, errr
	} else if fd, errr := os.Open(filePath); errr != nil {
		return []byte{}, fi.Size(), errr
	} else {
		defer fd.Close()
		size = fi.Size()

		hash160, hash512 := CreateHashBuffers(fd)

		dgst, err = CheckBufferHash(hash160, hash512, digest)

		return
	}
}

// ImportFile registers (ou updates if it already exists in database) the file in database.
func (e *Exercice) ImportFile(filePath string, origin string, digest []byte, digestshown []byte, disclaimer string, published bool) (interface{}, error) {
	if result512, size, err := checkFileHash(filePath, digest); !OptionalDigest && err != nil {
		return nil, err
	} else {
		dPath := strings.TrimPrefix(filePath, FilesDir)
		if f, err := e.GetFileByPath(dPath); err != nil {
			return e.AddFile(dPath, origin, path.Base(filePath), result512, digestshown, size, disclaimer, published)
		} else {
			// Don't need to update Path and Name, because they are related to dPath

			f.IdExercice = e.Id
			f.origin = origin
			f.Checksum = result512
			f.ChecksumShown = digestshown
			f.Size = size
			f.Disclaimer = disclaimer
			f.Published = published

			if _, err := f.Update(); err != nil {
				return nil, err
			} else {
				return f, nil
			}
		}
	}
}

// AddFile creates and fills a new struct File and registers it into the database.
func (e *Exercice) AddFile(path string, origin string, name string, checksum []byte, checksumshown []byte, size int64, disclaimer string, published bool) (*EFile, error) {
	if res, err := DBExec("INSERT INTO exercice_files (id_exercice, origin, path, name, cksum, cksum_shown, size, disclaimer, published) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", e.Id, origin, path, name, checksum, checksumshown, size, disclaimer, published); err != nil {
		return nil, err
	} else if fid, err := res.LastInsertId(); err != nil {
		return nil, err
	} else {
		return &EFile{fid, origin, path, e.Id, name, checksum, checksumshown, size, disclaimer, published}, nil
	}
}

// Update applies modifications back to the database.
func (f *EFile) Update() (int64, error) {
	if res, err := DBExec("UPDATE exercice_files SET id_exercice = ?, origin = ?, path = ?, name = ?, cksum = ?, cksum_shown = ?, size = ?, disclaimer = ?, published = ? WHERE id_file = ?", f.IdExercice, f.origin, f.Path, f.Name, f.Checksum, f.ChecksumShown, f.Size, f.Disclaimer, f.Published, f.Id); err != nil {
		return 0, err
	} else if nb, err := res.RowsAffected(); err != nil {
		return 0, err
	} else {
		return nb, err
	}
}

// Delete the file from the database.
func (f EFile) Delete() (int64, error) {
	if res, err := DBExec("DELETE FROM exercice_files WHERE id_file = ?", f.Id); err != nil {
		return 0, err
	} else if nb, err := res.RowsAffected(); err != nil {
		return 0, err
	} else {
		return nb, err
	}
}

// WipeFiles deletes (only in the database, not on disk) files coming with the challenge.
func (e Exercice) WipeFiles() (int64, error) {
	if _, err := DBExec("DELETE FROM exercice_files_okey_deps WHERE id_flag IN (SELECT id_flag FROM exercice_flags WHERE id_exercice = ?)", e.Id); err != nil {
		return 0, err
	} else if _, err := DBExec("DELETE FROM exercice_files_omcq_deps WHERE id_mcq IN (SELECT id_mcq FROM exercice_mcq WHERE id_exercice = ?)", e.Id); err != nil {
		return 0, err
	} else if res, err := DBExec("DELETE FROM exercice_files WHERE id_exercice = ?", e.Id); err != nil {
		return 0, err
	} else if nb, err := res.RowsAffected(); err != nil {
		return 0, err
	} else {
		return nb, err
	}
}

// ClearFiles removes all certificates from database (but not files on disks).
func ClearFiles() (int64, error) {
	if res, err := DBExec("DELETE FROM exercice_files"); err != nil {
		return 0, err
	} else if nb, err := res.RowsAffected(); err != nil {
		return 0, err
	} else {
		return nb, err
	}
}

// GetOrigin access the private field origin of the file.
func (f *EFile) GetOrigin() string {
	return f.origin
}

// AddDepend insert a new dependency to a given flag.
func (f *EFile) AddDepend(j Flag) (err error) {
	if k, ok := j.(*FlagKey); ok {
		_, err = DBExec("INSERT INTO exercice_files_okey_deps (id_file, id_flag) VALUES (?, ?)", f.Id, k.Id)
	} else if m, ok := j.(*MCQ); ok {
		_, err = DBExec("INSERT INTO exercice_files_omcq_deps (id_file, id_mcq) VALUES (?, ?)", f.Id, m.Id)
	} else {
		err = fmt.Errorf("dependancy type for flag (%T) not implemented for this file", j)
	}
	return
}

// DeleteDepend insert a new dependency to a given flag.
func (f *EFile) DeleteDepend(j Flag) (err error) {
	if k, ok := j.(*FlagKey); ok {
		_, err = DBExec("DELETE FROM exercice_files_okey_deps WHERE id_file = ? AND id_flag = ?", f.Id, k.Id)
	} else if m, ok := j.(*MCQ); ok {
		_, err = DBExec("DELETE FROM exercice_files_omcq_deps WHERE id_file = ? AND id_mcq = ?", f.Id, m.Id)
	} else {
		err = fmt.Errorf("dependancy type for flag (%T) not implemented for this file", j)
	}
	return
}

// GetDepends retrieve the flag's dependency list.
func (f *EFile) GetDepends() ([]Flag, error) {
	var deps = make([]Flag, 0)

	if rows, err := DBQuery("SELECT id_flag FROM exercice_files_okey_deps WHERE id_file = ?", f.Id); err != nil {
		return nil, err
	} else {
		defer rows.Close()

		for rows.Next() {
			var d int
			if err := rows.Scan(&d); err != nil {
				return nil, err
			}
			deps = append(deps, &FlagKey{d, f.IdExercice, 0, "", "", "", "", "", false, false, false, nil, false, []byte{}, 0, 0})
		}
		if err := rows.Err(); err != nil {
			return nil, err
		}
	}

	if rows, err := DBQuery("SELECT id_mcq FROM exercice_files_omcq_deps WHERE id_file = ?", f.Id); err != nil {
		return nil, err
	} else {
		defer rows.Close()

		for rows.Next() {
			var d int
			if err := rows.Scan(&d); err != nil {
				return nil, err
			}
			deps = append(deps, &MCQ{d, f.IdExercice, 0, "", []*MCQ_entry{}})
		}
		if err := rows.Err(); err != nil {
			return nil, err
		}
	}

	return deps, nil
}

// CheckFileOnDisk recalculates the hash of the file on disk.
func (f *EFile) CheckFileOnDisk() error {
	if _, size, err := checkFileHash(path.Join(FilesDir, f.Path), f.Checksum); err != nil {
		return err
	} else if size == 0 {
		if _, _, err := checkFileHash(path.Join(FilesDir, f.Path+".gz"), f.Checksum); err != nil {
			return errors.New("empty file")
		} else {
			return nil
		}
	} else {
		return nil
	}
}

// GunzipFileOnDisk gunzip a compressed file.
func (f *EFile) GunzipFileOnDisk() error {
	if !strings.HasSuffix(f.origin, ".gz") || strings.HasSuffix(f.origin, ".tar.gz") {
		return nil
	}

	fdIn, err := os.Open(path.Join(FilesDir, f.Path+".gz"))
	if err != nil {
		return err
	}
	defer fdIn.Close()

	fdOut, err := os.Create(path.Join(FilesDir, strings.TrimSuffix(f.Path, ".gz")))
	if err != nil {
		return err
	}
	defer fdOut.Close()

	gunzipfd, err := gzip.NewReader(fdIn)
	if err != nil {
		return err
	}
	defer gunzipfd.Close()

	_, err = io.Copy(fdOut, gunzipfd)
	if err != nil {
		return err
	}

	return nil
}