server/admin/sync/file.go

245 lines
6.7 KiB
Go
Raw Normal View History

package sync
import (
"bufio"
"bytes"
"encoding/base32"
"fmt"
"io"
2022-05-24 19:25:51 +00:00
"net/url"
"os"
"path"
"strings"
"srs.epita.fr/fic-server/libfic"
2018-02-03 01:03:08 +00:00
"golang.org/x/crypto/blake2b"
)
2018-03-09 18:07:08 +00:00
// Importer are abstract methods required to import challenges.
type Importer interface {
2018-03-09 18:07:08 +00:00
// Kind returns information about the Importer, for human interrest.
Kind() string
// Id returns information about the current state (commit id, ...).
Id() *string
// init performs the importer initialization.
Init() error
// sync tries to pull the latest modification of the underlying storage.
Sync() error
2023-05-03 08:54:02 +00:00
// Exists checks if the given location exists from the Importer point of view.
Exists(filename string) bool
2018-03-09 18:07:08 +00:00
// toURL gets the full path/URL to the given file, the Importer will look internaly (used for debuging purpose).
toURL(filename string) string
2018-05-11 23:08:37 +00:00
// importFile imports the file at the given URI, inside the global FILES/ directory.
// Then calls back the next function, with the downloaded location and the original URI.
// Callback return is forwarded.
importFile(URI string, next func(string, string) (interface{}, error)) (interface{}, error)
// getFileReader returns a reader to the requested file.
2023-11-25 16:13:31 +00:00
GetFile(filename string) (io.Reader, error)
2018-05-11 23:08:37 +00:00
// listDir returns a list of the files and subdirectories contained inside the directory at the given location.
2023-11-25 16:13:31 +00:00
ListDir(filename string) ([]string, error)
2018-05-11 23:08:37 +00:00
// stat returns many information about the given file: such as last modification date, size, ...
2023-11-25 16:13:31 +00:00
Stat(filename string) (os.FileInfo, error)
}
2022-05-22 23:39:35 +00:00
// DirectAccessImporter abstracts importer that support direct file access through a local path
type DirectAccessImporter interface {
GetLocalPath(p ...string) string
}
2022-05-24 19:25:51 +00:00
// ForgeLinkedImporter abstracts importer that are linked to a forge
type ForgeLinkedImporter interface {
GetThemeLink(th *fic.Theme) (*url.URL, error)
GetExerciceLink(e *fic.Exercice) (*url.URL, error)
}
2022-05-24 20:13:38 +00:00
// WritableImporter abstracts importer that we can also write on
type WritableImporter interface {
// writeFile write the given buffer to the file at the given location.
writeFile(filename string, reader io.Reader) error
}
2018-05-11 23:08:37 +00:00
// GlobalImporter stores the main importer instance to use for global imports.
var GlobalImporter Importer
2022-11-21 17:55:38 +00:00
// GetFileSize returns the size.
func GetFileSize(i Importer, URI string) (size int64, err error) {
2023-05-03 08:54:02 +00:00
if i.Exists(URI) {
2023-11-25 16:13:31 +00:00
if fi, err := i.Stat(URI); err != nil {
return 0, err
} else {
return fi.Size(), nil
}
}
dirname := path.Dir(URI)
2023-05-03 08:54:02 +00:00
if i.Exists(dirname) {
filename := path.Base(URI)
2023-11-25 16:13:31 +00:00
if files, err := i.ListDir(dirname); err != nil {
return size, err
} else {
for _, fname := range []string{filename, filename + "."} {
found := false
for _, file := range files {
if matched, _ := path.Match(fname+"[0-9][0-9]", file); matched {
found = true
2023-11-25 16:13:31 +00:00
if fi, err := i.Stat(path.Join(dirname, file)); err != nil {
return size, err
} else {
size += fi.Size()
}
}
}
if found {
return size, nil
}
}
}
}
2020-04-15 05:39:38 +00:00
return size, fmt.Errorf("%q: no such file or directory", URI)
}
// GetFile helps to manage huge file transfert by concatenating splitted (with split(1)) files.
2022-11-21 13:38:16 +00:00
func GetFile(i Importer, URI string) (io.Reader, func(), error) {
// Import file if it exists
2023-05-03 08:54:02 +00:00
if i.Exists(URI) {
2023-11-25 16:13:31 +00:00
fd, err := i.GetFile(URI)
2022-11-21 13:38:16 +00:00
return fd, func() {
if fdc, ok := fd.(io.ReadCloser); ok {
fdc.Close()
}
}, err
}
// Try to find file parts
dirname := path.Dir(URI)
2023-05-03 08:54:02 +00:00
if i.Exists(dirname) {
filename := path.Base(URI)
2023-11-25 16:13:31 +00:00
if files, err := i.ListDir(dirname); err != nil {
2022-11-21 13:38:16 +00:00
return nil, nil, err
} else {
2022-11-21 13:38:16 +00:00
var readers []io.Reader
for _, fname := range []string{filename, filename + "."} {
for _, file := range files {
if matched, _ := path.Match(fname+"[0-9][0-9]", file); matched {
2023-11-25 16:13:31 +00:00
fd, err := i.GetFile(path.Join(dirname, file))
if err != nil {
2022-11-21 13:38:16 +00:00
// Close already opened files to avoid leaks
for _, rd := range readers {
if rdc, ok := rd.(io.ReadCloser); ok {
rdc.Close()
}
}
return nil, nil, err
}
2022-11-21 13:38:16 +00:00
readers = append(readers, fd)
}
}
2022-11-21 13:38:16 +00:00
if len(readers) > 0 {
return io.MultiReader(readers...), func() {
for _, rd := range readers {
if rdc, ok := rd.(io.ReadCloser); ok {
rdc.Close()
}
}
}, nil
}
}
}
}
2022-11-21 13:38:16 +00:00
return nil, nil, fmt.Errorf("%q: no such file or directory", URI)
}
// GetFileContent retrieves the content of the given text file.
func GetFileContent(i Importer, URI string) (string, error) {
2022-11-21 13:38:16 +00:00
if fd, closer, err := GetFile(i, URI); err != nil {
return "", err
} else {
2022-11-21 13:38:16 +00:00
defer closer()
buffd := bufio.NewReader(fd)
// Ensure we read UTF-8 content.
buf := make([]rune, 0)
2022-11-21 13:38:16 +00:00
for b, _, err := buffd.ReadRune(); err == nil; b, _, err = buffd.ReadRune() {
buf = append(buf, b)
}
2021-11-22 14:35:07 +00:00
if len(buf) == 0 {
return "", fmt.Errorf("File is empty")
}
return strings.TrimSpace(string(buf)), nil
}
}
// GetDestinationFilePath generates the destination path, from the URI.
2018-03-09 18:07:08 +00:00
// This function permits to obfusce to player the original URI.
// Theoricaly, changing the import method doesn't change destination URI.
func GetDestinationFilePath(URI string, filename *string) string {
2023-10-13 14:05:15 +00:00
if filename == nil {
tmp := path.Base(URI)
filename = &tmp
}
hash := blake2b.Sum512([]byte(URI))
2023-10-13 14:05:15 +00:00
return path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), *filename)
}
2022-06-06 18:42:46 +00:00
func importFile(i Importer, URI string, dest string) error {
if err := os.MkdirAll(path.Dir(dest), 0751); err != nil {
2022-06-06 18:42:46 +00:00
return err
}
// Write file
if fdto, err := os.Create(dest); err != nil {
return err
} else {
defer fdto.Close()
2022-11-21 13:38:16 +00:00
if fdfrom, closer, err := GetFile(i, URI); err != nil {
2022-06-06 18:42:46 +00:00
os.Remove(dest)
return err
2022-11-21 13:38:16 +00:00
} else {
defer closer()
_, err = io.Copy(fdto, fdfrom)
return err
2022-06-06 18:42:46 +00:00
}
}
}
2018-03-09 18:07:08 +00:00
// ImportFile imports the file at the given URI, using helpers of the given Importer.
// After import, next is called with relative path where the file has been saved and the original URI.
func ImportFile(i Importer, URI string, next func(string, string) (interface{}, error)) (interface{}, error) {
dest := GetDestinationFilePath(URI, nil)
// If the present file is still valide, don't erase it
if _, err := os.Stat(dest); !os.IsNotExist(err) {
if r, err := next(dest, URI); err == nil {
return r, err
}
}
2022-06-06 18:42:46 +00:00
if err := importFile(i, URI, dest); err != nil {
return nil, err
}
return next(dest, URI)
}
2022-05-24 20:13:38 +00:00
// WriteFileContent save the given content to the given text file.
func WriteFileContent(i Importer, URI string, content []byte) error {
if wi, ok := i.(WritableImporter); ok {
return wi.writeFile(URI, bytes.NewReader(content))
} else {
return fmt.Errorf("%t is not capable of writing", i)
}
}