admin: Implement sychronization backends
We are now able, depending on configuration, to retrieve files from either WebDAV or local file system.
This commit is contained in:
parent
6237f7755a
commit
8f7de926d3
@ -2,10 +2,12 @@ package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"srs.epita.fr/fic-server/libfic"
|
||||
"srs.epita.fr/fic-server/admin/sync"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
)
|
||||
@ -125,7 +127,6 @@ type uploadedHint struct {
|
||||
Content string
|
||||
Cost int64
|
||||
URI string
|
||||
Path string
|
||||
}
|
||||
|
||||
func createExerciceHint(exercice fic.Exercice, body []byte) (interface{}, error) {
|
||||
@ -136,9 +137,9 @@ func createExerciceHint(exercice fic.Exercice, body []byte) (interface{}, error)
|
||||
|
||||
if len(uh.Content) != 0 {
|
||||
return exercice.AddHint(uh.Title, uh.Content, uh.Cost)
|
||||
} else if len(uh.Path) != 0 || len(uh.URI) != 0 {
|
||||
return importFile(uploadedFile{Path: uh.Path, URI: uh.URI},
|
||||
func(filePath string, origin string, digest []byte) (interface{}, error) {
|
||||
} else if len(uh.URI) != 0 {
|
||||
return sync.ImportFile(uh.URI,
|
||||
func(filePath string, origin string) (interface{}, error) {
|
||||
return exercice.AddHint(uh.Title, "$FILES" + strings.TrimPrefix(filePath, fic.FilesDir), uh.Cost)
|
||||
})
|
||||
} else {
|
||||
@ -209,7 +210,14 @@ func createExerciceFile(exercice fic.Exercice, body []byte) (interface{}, error)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return importFile(uf, exercice.ImportFile)
|
||||
return sync.ImportFile(uf.URI,
|
||||
func(filePath string, origin string) (interface{}, error) {
|
||||
if digest, err := hex.DecodeString(uf.Digest); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
return exercice.ImportFile(filePath, origin, digest)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func showExerciceFile(file fic.EFile, body []byte) (interface{}, error) {
|
||||
|
@ -1,154 +1,8 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/base32"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"srs.epita.fr/fic-server/libfic"
|
||||
|
||||
"github.com/dchest/blake2b"
|
||||
)
|
||||
|
||||
var CloudDAVBase string = "https://srs.epita.fr/owncloud/remote.php/webdav/FIC 2018"
|
||||
var CloudUsername string = "fic"
|
||||
var CloudPassword string = ""
|
||||
var RapidImport bool = false
|
||||
import ()
|
||||
|
||||
type uploadedFile struct {
|
||||
URI string
|
||||
Digest string
|
||||
Path string
|
||||
Parts []string
|
||||
}
|
||||
|
||||
func importFile(uf uploadedFile, next func(string, string, []byte) (interface{}, error)) (interface{}, error) {
|
||||
var hash [blake2b.Size]byte
|
||||
var logStr string
|
||||
var fromURI string
|
||||
var getFile func(string) error
|
||||
|
||||
if uf.URI != "" && len(uf.Parts) > 0 {
|
||||
hash = blake2b.Sum512([]byte(uf.URI))
|
||||
logStr = fmt.Sprintf("Import file from Cloud: %s =>", uf.Parts)
|
||||
fromURI = uf.URI
|
||||
getFile = func(dest string) error {
|
||||
if fdto, err := os.Create(dest); err != nil {
|
||||
return err
|
||||
} else {
|
||||
writer := bufio.NewWriter(fdto)
|
||||
for _, partname := range uf.Parts {
|
||||
if err := getCloudPart(partname, writer); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
fdto.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
} else if uf.URI != "" {
|
||||
hash = blake2b.Sum512([]byte(uf.URI))
|
||||
logStr = "Import file from Cloud: " + uf.URI + " =>"
|
||||
fromURI = uf.URI
|
||||
getFile = func(dest string) error { return getCloudFile(uf.URI, dest) }
|
||||
} else if uf.Path != "" && len(uf.Parts) > 0 {
|
||||
hash = blake2b.Sum512([]byte(uf.Path))
|
||||
logStr = fmt.Sprintf("Import file from local FS: %s =>", uf.Parts)
|
||||
fromURI = uf.Path
|
||||
getFile = func(dest string) error {
|
||||
if fdto, err := os.Create(dest); err != nil {
|
||||
return err
|
||||
} else {
|
||||
writer := bufio.NewWriter(fdto)
|
||||
for _, partname := range uf.Parts {
|
||||
if fdfrm, err := os.Open(partname); err != nil {
|
||||
return err
|
||||
} else {
|
||||
reader := bufio.NewReader(fdfrm)
|
||||
reader.WriteTo(writer)
|
||||
writer.Flush()
|
||||
fdfrm.Close()
|
||||
}
|
||||
}
|
||||
fdto.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
} else if uf.Path != "" {
|
||||
hash = blake2b.Sum512([]byte(uf.Path))
|
||||
logStr = "Import file from local FS: " + uf.Path + " =>"
|
||||
fromURI = uf.Path
|
||||
getFile = func(dest string) error { return os.Symlink(uf.Path, dest) }
|
||||
} else {
|
||||
return nil, errors.New("URI or path not filled")
|
||||
}
|
||||
|
||||
pathname := path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(0).EncodeToString(hash[:])), path.Base(fromURI))
|
||||
|
||||
// Remove the file if it exists
|
||||
// TODO: check if this is symlink => remove to avoid File not found error after, because the file is writen at the adresse pointed.
|
||||
if _, err := os.Stat(pathname); !os.IsNotExist(err) && !RapidImport {
|
||||
if err := os.Remove(pathname); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := os.Stat(pathname); os.IsNotExist(err) {
|
||||
log.Println(logStr, pathname)
|
||||
if err := os.MkdirAll(path.Dir(pathname), 0777); err != nil {
|
||||
return nil, err
|
||||
} else if err := getFile(pathname); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if digest, err := hex.DecodeString(uf.Digest); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
return next(pathname, fromURI, digest)
|
||||
}
|
||||
}
|
||||
|
||||
func getCloudFile(pathname string, dest string) error {
|
||||
if fd, err := os.Create(dest); err != nil {
|
||||
return err
|
||||
} else {
|
||||
defer fd.Close()
|
||||
|
||||
writer := bufio.NewWriter(fd)
|
||||
if err := getCloudPart(pathname, writer); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getCloudPart(pathname string, writer *bufio.Writer) error {
|
||||
client := http.Client{}
|
||||
if req, err := http.NewRequest("GET", CloudDAVBase+pathname, nil); err != nil {
|
||||
return err
|
||||
} else {
|
||||
req.SetBasicAuth(CloudUsername, CloudPassword)
|
||||
if resp, err := client.Do(req); err != nil {
|
||||
return err
|
||||
} else {
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return errors.New(resp.Status)
|
||||
} else {
|
||||
reader := bufio.NewReader(resp.Body)
|
||||
reader.WriteTo(writer)
|
||||
writer.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
@ -13,6 +13,7 @@ import (
|
||||
"text/template"
|
||||
|
||||
"srs.epita.fr/fic-server/admin/api"
|
||||
"srs.epita.fr/fic-server/admin/sync"
|
||||
"srs.epita.fr/fic-server/libfic"
|
||||
"srs.epita.fr/fic-server/settings"
|
||||
)
|
||||
@ -63,13 +64,13 @@ func StripPrefix(prefix string, h http.Handler) http.Handler {
|
||||
func main() {
|
||||
// Read paremeters from environment
|
||||
if v, exists := os.LookupEnv("FICCLOUD_URL"); exists {
|
||||
api.CloudDAVBase = v
|
||||
sync.CloudDAVBase = v
|
||||
}
|
||||
if v, exists := os.LookupEnv("FICCLOUD_USER"); exists {
|
||||
api.CloudUsername = v
|
||||
sync.CloudUsername = v
|
||||
}
|
||||
if v, exists := os.LookupEnv("FICCLOUD_PASS"); exists {
|
||||
api.CloudPassword = v
|
||||
sync.CloudPassword = v
|
||||
}
|
||||
|
||||
// Read parameters from command line
|
||||
@ -81,11 +82,10 @@ func main() {
|
||||
flag.StringVar(&api.TeamsDir, "teams", "./TEAMS", "Base directory where save teams JSON files")
|
||||
flag.StringVar(&settings.SettingsDir, "settings", settings.SettingsDir, "Base directory where load and save settings")
|
||||
flag.StringVar(&fic.FilesDir, "files", fic.FilesDir, "Base directory where found challenges files, local part")
|
||||
flag.StringVar(&api.CloudDAVBase, "clouddav", api.CloudDAVBase,
|
||||
flag.StringVar(&sync.CloudDAVBase, "clouddav", sync.CloudDAVBase,
|
||||
"Base directory where found challenges files, cloud part")
|
||||
flag.StringVar(&api.CloudUsername, "clouduser", api.CloudUsername, "Username used to sync")
|
||||
flag.StringVar(&api.CloudPassword, "cloudpass", api.CloudPassword, "Password used to sync")
|
||||
flag.BoolVar(&api.RapidImport, "rapidimport", api.RapidImport, "Don't try to reimport an existing file")
|
||||
flag.StringVar(&sync.CloudUsername, "clouduser", sync.CloudUsername, "Username used to sync")
|
||||
flag.StringVar(&sync.CloudPassword, "cloudpass", sync.CloudPassword, "Password used to sync")
|
||||
flag.BoolVar(&fic.OptionalDigest, "optionaldigest", fic.OptionalDigest, "Is the digest required when importing files?")
|
||||
flag.BoolVar(&fic.StrongDigest, "strongdigest", fic.StrongDigest, "Are BLAKE2b digests required instead of SHA-1 or BLAKE2b?")
|
||||
flag.Parse()
|
||||
|
98
admin/sync/file.go
Normal file
98
admin/sync/file.go
Normal file
@ -0,0 +1,98 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/base32"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"srs.epita.fr/fic-server/libfic"
|
||||
|
||||
"github.com/dchest/blake2b"
|
||||
)
|
||||
|
||||
type Importer interface {
|
||||
Kind() string
|
||||
exists(filename string) bool
|
||||
toURL(filename string) string
|
||||
getFile(filename string, writer *bufio.Writer) error
|
||||
listDir(filename string) ([]string, error)
|
||||
}
|
||||
|
||||
var GlobalImporter Importer
|
||||
|
||||
func getFile(i Importer, URI string, writer *bufio.Writer) error {
|
||||
// Import file if it exists
|
||||
if i.exists(URI) {
|
||||
return i.getFile(URI, writer)
|
||||
}
|
||||
|
||||
// Try to find file parts
|
||||
dirname := path.Dir(URI)
|
||||
if i.exists(dirname) {
|
||||
filename := path.Base(URI)
|
||||
if files, err := i.listDir(dirname); err != nil {
|
||||
return err
|
||||
} else {
|
||||
for _, fname := range []string{filename, filename + "."} {
|
||||
found := false
|
||||
for _, file := range files {
|
||||
if matched, _ := path.Match(fname + "[0-9][0-9]", file); matched {
|
||||
found = true
|
||||
if err := i.getFile(path.Join(dirname, file), writer); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if found {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return errors.New(fmt.Sprintf("%q: no such file or directory", i.toURL(URI)))
|
||||
}
|
||||
|
||||
func ImportFile(URI string, next func(string, string) (interface{}, error)) (interface{}, error) {
|
||||
hash := blake2b.Sum512([]byte(URI))
|
||||
dest := path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), path.Base(URI))
|
||||
|
||||
// If the present file is still valide, don't erase it
|
||||
if _, err := os.Stat(dest); !os.IsNotExist(err) {
|
||||
if r, err := next(dest, URI); err == nil {
|
||||
return r, err
|
||||
}
|
||||
|
||||
if err := os.Remove(dest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure no more file is registered with this path
|
||||
if f, err := fic.GetFileByPath(dest); err == nil {
|
||||
f.Delete()
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(path.Dir(dest), 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Write file
|
||||
if fdto, err := os.Create(dest); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
defer fdto.Close()
|
||||
writer := bufio.NewWriter(fdto)
|
||||
if err := getFile(GlobalImporter, URI, writer); err != nil {
|
||||
os.Remove(dest)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return next(dest, URI)
|
||||
}
|
96
admin/sync/importer_cloud.go
Normal file
96
admin/sync/importer_cloud.go
Normal file
@ -0,0 +1,96 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
|
||||
"github.com/studio-b12/gowebdav"
|
||||
)
|
||||
|
||||
var CloudDAVBase string = "https://srs.epita.fr/owncloud/remote.php/webdav/FIC 2018"
|
||||
var CloudUsername string = "fic"
|
||||
var CloudPassword string = ""
|
||||
|
||||
type CloudImporter struct {
|
||||
baseDAV url.URL
|
||||
username string
|
||||
password string
|
||||
}
|
||||
|
||||
func NewCloudImporter(baseDAV string, username string, password string) (*CloudImporter, error) {
|
||||
if r, err := url.Parse(baseDAV); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
return &CloudImporter{*r, username, password}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (i CloudImporter) Kind() string {
|
||||
return "cloud file importer: " + i.baseDAV.String()
|
||||
}
|
||||
|
||||
func (i CloudImporter) exists(filename string) bool {
|
||||
fullURL := i.baseDAV
|
||||
fullURL.Path = path.Join(fullURL.Path, filename)
|
||||
|
||||
client := http.Client{}
|
||||
if req, err := http.NewRequest("HEAD", fullURL.String(), nil); err == nil {
|
||||
req.SetBasicAuth(i.username, i.password)
|
||||
|
||||
if resp, err := client.Do(req); err == nil {
|
||||
defer resp.Body.Close()
|
||||
|
||||
return resp.StatusCode == http.StatusOK
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (i CloudImporter) toURL(filename string) string {
|
||||
fullURL := i.baseDAV
|
||||
fullURL.Path = path.Join(fullURL.Path, filename)
|
||||
return fullURL.String()
|
||||
}
|
||||
|
||||
func (i CloudImporter) getFile(filename string, writer *bufio.Writer) error {
|
||||
fullURL := i.baseDAV
|
||||
fullURL.Path = path.Join(fullURL.Path, filename)
|
||||
|
||||
client := http.Client{}
|
||||
if req, err := http.NewRequest("GET", fullURL.String(), nil); err != nil {
|
||||
return err
|
||||
} else {
|
||||
req.SetBasicAuth(i.username, i.password)
|
||||
if resp, err := client.Do(req); err != nil {
|
||||
return err
|
||||
} else {
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return errors.New(resp.Status)
|
||||
} else {
|
||||
reader := bufio.NewReader(resp.Body)
|
||||
reader.WriteTo(writer)
|
||||
writer.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (i CloudImporter) listDir(filename string) ([]string, error) {
|
||||
client := gowebdav.NewClient(i.baseDAV.String(), i.username, i.password)
|
||||
|
||||
if files, err := client.ReadDir(filename); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
res := make([]string, 0)
|
||||
for _, file := range files {
|
||||
res = append(res, file.Name())
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
}
|
49
admin/sync/importer_localfs.go
Normal file
49
admin/sync/importer_localfs.go
Normal file
@ -0,0 +1,49 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
)
|
||||
|
||||
type LocalImporter struct {
|
||||
Base string
|
||||
}
|
||||
|
||||
func (i LocalImporter) Kind() string {
|
||||
return "local file importer: " + i.Base
|
||||
}
|
||||
|
||||
func (i LocalImporter) exists(filename string) bool {
|
||||
_, err := os.Stat(path.Join(i.Base, filename))
|
||||
return !os.IsNotExist(err)
|
||||
}
|
||||
|
||||
func (i LocalImporter) toURL(filename string) string {
|
||||
return path.Join(i.Base, filename)
|
||||
}
|
||||
|
||||
func (i LocalImporter) getFile(filename string, writer *bufio.Writer) error {
|
||||
if fd, err := os.Open(path.Join(i.Base, filename)); err != nil {
|
||||
return err
|
||||
} else {
|
||||
defer fd.Close()
|
||||
reader := bufio.NewReader(fd)
|
||||
reader.WriteTo(writer)
|
||||
writer.Flush()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (i LocalImporter) listDir(filename string) ([]string, error) {
|
||||
if files, err := ioutil.ReadDir(path.Join(i.Base, filename)); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
res := make([]string, 0)
|
||||
for _, file := range files {
|
||||
res = append(res, file.Name())
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
}
|
@ -135,7 +135,23 @@ func (e Exercice) ImportFile(filePath string, origin string, digest []byte) (int
|
||||
}
|
||||
}
|
||||
|
||||
return e.AddFile(strings.TrimPrefix(filePath, FilesDir), origin, path.Base(filePath), result512, fi.Size())
|
||||
dPath := strings.TrimPrefix(filePath, FilesDir)
|
||||
if f, err := e.GetFileByPath(dPath); err != nil {
|
||||
return e.AddFile(dPath, origin, path.Base(filePath), result512, fi.Size())
|
||||
} else {
|
||||
// Don't need to update Path and Name, because they are related to dPath
|
||||
|
||||
f.IdExercice = e.Id
|
||||
f.origin = origin
|
||||
f.Checksum = result512
|
||||
f.Size = fi.Size()
|
||||
|
||||
if _, err := f.Update(); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user