2017-11-27 01:45:33 +00:00
|
|
|
package sync
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bufio"
|
2017-12-09 00:20:30 +00:00
|
|
|
"bytes"
|
2017-11-27 01:45:33 +00:00
|
|
|
"encoding/base32"
|
|
|
|
"fmt"
|
2017-12-09 00:20:30 +00:00
|
|
|
"io"
|
2017-11-27 01:45:33 +00:00
|
|
|
"os"
|
|
|
|
"path"
|
|
|
|
"strings"
|
|
|
|
|
|
|
|
"srs.epita.fr/fic-server/libfic"
|
|
|
|
|
2018-02-03 01:03:08 +00:00
|
|
|
"golang.org/x/crypto/blake2b"
|
2017-11-27 01:45:33 +00:00
|
|
|
)
|
|
|
|
|
2018-03-09 18:07:08 +00:00
|
|
|
// Importer are abstract methods required to import challenges.
|
2017-11-27 01:45:33 +00:00
|
|
|
type Importer interface {
|
2018-03-09 18:07:08 +00:00
|
|
|
// Kind returns information about the Importer, for human interrest.
|
2017-11-27 01:45:33 +00:00
|
|
|
Kind() string
|
2022-01-20 14:29:49 +00:00
|
|
|
// Id returns information about the current state (commit id, ...).
|
|
|
|
Id() *string
|
2021-10-26 17:56:40 +00:00
|
|
|
// init performs the importer initialization.
|
|
|
|
Init() error
|
|
|
|
// sync tries to pull the latest modification of the underlying storage.
|
|
|
|
Sync() error
|
2018-03-09 18:07:08 +00:00
|
|
|
// exists checks if the given location exists from the Importer point of view.
|
2017-11-27 01:45:33 +00:00
|
|
|
exists(filename string) bool
|
2018-03-09 18:07:08 +00:00
|
|
|
// toURL gets the full path/URL to the given file, the Importer will look internaly (used for debuging purpose).
|
2017-11-27 01:45:33 +00:00
|
|
|
toURL(filename string) string
|
2018-05-11 23:08:37 +00:00
|
|
|
// importFile imports the file at the given URI, inside the global FILES/ directory.
|
|
|
|
// Then calls back the next function, with the downloaded location and the original URI.
|
|
|
|
// Callback return is forwarded.
|
2017-12-12 06:13:38 +00:00
|
|
|
importFile(URI string, next func(string, string) (interface{}, error)) (interface{}, error)
|
2018-05-11 23:08:37 +00:00
|
|
|
// getFile write to the given buffer, the file at the given location.
|
2017-11-27 01:45:33 +00:00
|
|
|
getFile(filename string, writer *bufio.Writer) error
|
2018-05-11 23:08:37 +00:00
|
|
|
// listDir returns a list of the files and subdirectories contained inside the directory at the given location.
|
2017-11-27 01:45:33 +00:00
|
|
|
listDir(filename string) ([]string, error)
|
2018-05-11 23:08:37 +00:00
|
|
|
// stat returns many information about the given file: such as last modification date, size, ...
|
2018-01-07 21:21:13 +00:00
|
|
|
stat(filename string) (os.FileInfo, error)
|
2017-11-27 01:45:33 +00:00
|
|
|
}
|
|
|
|
|
2018-05-11 23:08:37 +00:00
|
|
|
// GlobalImporter stores the main importer instance to use for global imports.
|
2017-11-27 01:45:33 +00:00
|
|
|
var GlobalImporter Importer
|
|
|
|
|
2019-01-19 00:13:17 +00:00
|
|
|
// getFileSize returns the size.
|
|
|
|
func getFileSize(i Importer, URI string) (size int64, err error) {
|
|
|
|
if i.exists(URI) {
|
|
|
|
if fi, err := i.stat(URI); err != nil {
|
|
|
|
return 0, err
|
|
|
|
} else {
|
|
|
|
return fi.Size(), nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
dirname := path.Dir(URI)
|
|
|
|
if i.exists(dirname) {
|
|
|
|
filename := path.Base(URI)
|
|
|
|
if files, err := i.listDir(dirname); err != nil {
|
|
|
|
return size, err
|
|
|
|
} else {
|
|
|
|
for _, fname := range []string{filename, filename + "."} {
|
|
|
|
found := false
|
|
|
|
for _, file := range files {
|
2019-07-05 17:07:28 +00:00
|
|
|
if matched, _ := path.Match(fname+"[0-9][0-9]", file); matched {
|
2019-01-19 00:13:17 +00:00
|
|
|
found = true
|
|
|
|
if fi, err := i.stat(path.Join(dirname, file)); err != nil {
|
|
|
|
return size, err
|
|
|
|
} else {
|
|
|
|
size += fi.Size()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if found {
|
|
|
|
return size, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-15 05:39:38 +00:00
|
|
|
return size, fmt.Errorf("%q: no such file or directory", URI)
|
2019-01-19 00:13:17 +00:00
|
|
|
}
|
|
|
|
|
2018-05-11 23:08:37 +00:00
|
|
|
// getFile helps to manage huge file transfert by concatenating splitted (with split(1)) files.
|
2017-11-27 01:45:33 +00:00
|
|
|
func getFile(i Importer, URI string, writer *bufio.Writer) error {
|
|
|
|
// Import file if it exists
|
|
|
|
if i.exists(URI) {
|
|
|
|
return i.getFile(URI, writer)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Try to find file parts
|
|
|
|
dirname := path.Dir(URI)
|
|
|
|
if i.exists(dirname) {
|
|
|
|
filename := path.Base(URI)
|
|
|
|
if files, err := i.listDir(dirname); err != nil {
|
|
|
|
return err
|
|
|
|
} else {
|
|
|
|
for _, fname := range []string{filename, filename + "."} {
|
|
|
|
found := false
|
|
|
|
for _, file := range files {
|
2019-07-05 17:07:28 +00:00
|
|
|
if matched, _ := path.Match(fname+"[0-9][0-9]", file); matched {
|
2017-11-27 01:45:33 +00:00
|
|
|
found = true
|
|
|
|
if err := i.getFile(path.Join(dirname, file), writer); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if found {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-15 05:39:38 +00:00
|
|
|
return fmt.Errorf("%q: no such file or directory", URI)
|
2017-11-27 01:45:33 +00:00
|
|
|
}
|
|
|
|
|
2018-05-11 23:08:37 +00:00
|
|
|
// getFileContent retrieves the content of the given text file.
|
2017-12-09 00:20:30 +00:00
|
|
|
func getFileContent(i Importer, URI string) (string, error) {
|
|
|
|
cnt := bytes.Buffer{}
|
|
|
|
|
|
|
|
if err := getFile(i, URI, bufio.NewWriter(io.Writer(&cnt))); err != nil {
|
|
|
|
return "", err
|
|
|
|
} else {
|
2018-05-12 10:15:00 +00:00
|
|
|
// Ensure we read UTF-8 content.
|
|
|
|
buf := make([]rune, 0)
|
|
|
|
for b, _, err := cnt.ReadRune(); err == nil; b, _, err = cnt.ReadRune() {
|
|
|
|
buf = append(buf, b)
|
|
|
|
}
|
|
|
|
|
2021-11-22 14:35:07 +00:00
|
|
|
if len(buf) == 0 {
|
|
|
|
return "", fmt.Errorf("File is empty")
|
|
|
|
}
|
|
|
|
|
2018-05-12 10:15:00 +00:00
|
|
|
return strings.TrimSpace(string(buf)), nil
|
2017-12-09 00:20:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-09 18:07:08 +00:00
|
|
|
// getDestinationFilePath generates the destination path, from the URI.
|
|
|
|
// This function permits to obfusce to player the original URI.
|
|
|
|
// Theoricaly, changing the import method doesn't change destination URI.
|
2017-12-12 05:22:22 +00:00
|
|
|
func getDestinationFilePath(URI string) string {
|
2017-11-27 01:45:33 +00:00
|
|
|
hash := blake2b.Sum512([]byte(URI))
|
2017-12-12 05:22:22 +00:00
|
|
|
return path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), path.Base(URI))
|
|
|
|
}
|
|
|
|
|
2018-03-09 18:07:08 +00:00
|
|
|
// ImportFile imports the file at the given URI, using helpers of the given Importer.
|
|
|
|
// After import, next is called with relative path where the file has been saved and the original URI.
|
2017-12-12 05:22:22 +00:00
|
|
|
func ImportFile(i Importer, URI string, next func(string, string) (interface{}, error)) (interface{}, error) {
|
|
|
|
dest := getDestinationFilePath(URI)
|
2017-11-27 01:45:33 +00:00
|
|
|
|
|
|
|
// If the present file is still valide, don't erase it
|
|
|
|
if _, err := os.Stat(dest); !os.IsNotExist(err) {
|
|
|
|
if r, err := next(dest, URI); err == nil {
|
|
|
|
return r, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := os.MkdirAll(path.Dir(dest), 0755); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Write file
|
|
|
|
if fdto, err := os.Create(dest); err != nil {
|
|
|
|
return nil, err
|
|
|
|
} else {
|
|
|
|
defer fdto.Close()
|
|
|
|
writer := bufio.NewWriter(fdto)
|
2017-12-12 05:22:22 +00:00
|
|
|
if err := getFile(i, URI, writer); err != nil {
|
2017-11-27 01:45:33 +00:00
|
|
|
os.Remove(dest)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return next(dest, URI)
|
|
|
|
}
|