admin: Implement sychronization backends

We are now able, depending on configuration, to retrieve files from either WebDAV or local file system.

admin/api/exercice.go

@@ -2,10 +2,12 @@ package api
 
 import (
 	"encoding/json"
+	"encoding/hex"
 	"errors"
 	"strings"
 
 	"srs.epita.fr/fic-server/libfic"
+	"srs.epita.fr/fic-server/admin/sync"
 
 	"github.com/julienschmidt/httprouter"
 )
@@ -125,7 +127,6 @@ type uploadedHint struct {
 	Content    string
 	Cost       int64
 	URI        string
-	Path       string
 }
 
 func createExerciceHint(exercice fic.Exercice, body []byte) (interface{}, error) {
@@ -136,9 +137,9 @@ func createExerciceHint(exercice fic.Exercice, body []byte) (interface{}, error)
 
 	if len(uh.Content) != 0 {
 		return exercice.AddHint(uh.Title, uh.Content, uh.Cost)
-	} else if len(uh.Path) != 0 || len(uh.URI) != 0 {
-		return importFile(uploadedFile{Path: uh.Path, URI: uh.URI},
-			func(filePath string, origin string, digest []byte) (interface{}, error) {
+	} else if len(uh.URI) != 0 {
+		return sync.ImportFile(uh.URI,
+			func(filePath string, origin string) (interface{}, error) {
 				return exercice.AddHint(uh.Title, "$FILES" + strings.TrimPrefix(filePath, fic.FilesDir), uh.Cost)
 			})
 	} else {
@@ -209,7 +210,14 @@ func createExerciceFile(exercice fic.Exercice, body []byte) (interface{}, error)
 		return nil, err
 	}
 
-	return importFile(uf, exercice.ImportFile)
+	return sync.ImportFile(uf.URI,
+		func(filePath string, origin string) (interface{}, error) {
+			if digest, err := hex.DecodeString(uf.Digest); err != nil {
+				return nil, err
+			} else {
+				return exercice.ImportFile(filePath, origin, digest)
+			}
+		})
 }
 
 func showExerciceFile(file fic.EFile, body []byte) (interface{}, error) {

admin/api/file.go

@@ -1,154 +1,8 @@
 package api
 
-import (
-	"bufio"
-	"encoding/base32"
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"log"
-	"net/http"
-	"os"
-	"path"
-	"strings"
-
-	"srs.epita.fr/fic-server/libfic"
-
-	"github.com/dchest/blake2b"
-)
-
-var CloudDAVBase string = "https://srs.epita.fr/owncloud/remote.php/webdav/FIC 2018"
-var CloudUsername string = "fic"
-var CloudPassword string = ""
-var RapidImport bool = false
+import ()
 
 type uploadedFile struct {
 	URI    string
 	Digest string
-	Path   string
-	Parts  []string
-}
-
-func importFile(uf uploadedFile, next func(string, string, []byte) (interface{}, error)) (interface{}, error) {
-	var hash [blake2b.Size]byte
-	var logStr string
-	var fromURI string
-	var getFile func(string) error
-
-	if uf.URI != "" && len(uf.Parts) > 0 {
-		hash = blake2b.Sum512([]byte(uf.URI))
-		logStr = fmt.Sprintf("Import file from Cloud: %s =>", uf.Parts)
-		fromURI = uf.URI
-		getFile = func(dest string) error {
-			if fdto, err := os.Create(dest); err != nil {
-				return err
-			} else {
-				writer := bufio.NewWriter(fdto)
-				for _, partname := range uf.Parts {
-					if err := getCloudPart(partname, writer); err != nil {
-						return err
-					}
-				}
-				fdto.Close()
-			}
-			return nil
-		}
-	} else if uf.URI != "" {
-		hash = blake2b.Sum512([]byte(uf.URI))
-		logStr = "Import file from Cloud: " + uf.URI + " =>"
-		fromURI = uf.URI
-		getFile = func(dest string) error { return getCloudFile(uf.URI, dest) }
-	} else if uf.Path != "" && len(uf.Parts) > 0 {
-		hash = blake2b.Sum512([]byte(uf.Path))
-		logStr = fmt.Sprintf("Import file from local FS: %s =>", uf.Parts)
-		fromURI = uf.Path
-		getFile = func(dest string) error {
-			if fdto, err := os.Create(dest); err != nil {
-				return err
-			} else {
-				writer := bufio.NewWriter(fdto)
-				for _, partname := range uf.Parts {
-					if fdfrm, err := os.Open(partname); err != nil {
-						return err
-					} else {
-						reader := bufio.NewReader(fdfrm)
-						reader.WriteTo(writer)
-						writer.Flush()
-						fdfrm.Close()
-					}
-				}
-				fdto.Close()
-			}
-			return nil
-		}
-	} else if uf.Path != "" {
-		hash = blake2b.Sum512([]byte(uf.Path))
-		logStr = "Import file from local FS: " + uf.Path + " =>"
-		fromURI = uf.Path
-		getFile = func(dest string) error { return os.Symlink(uf.Path, dest) }
-	} else {
-		return nil, errors.New("URI or path not filled")
-	}
-
-	pathname := path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(0).EncodeToString(hash[:])), path.Base(fromURI))
-
-	// Remove the file if it exists
-	// TODO: check if this is symlink => remove to avoid File not found error after, because the file is writen at the adresse pointed.
-	if _, err := os.Stat(pathname); !os.IsNotExist(err) && !RapidImport {
-		if err := os.Remove(pathname); err != nil {
-			return nil, err
-		}
-	}
-
-	if _, err := os.Stat(pathname); os.IsNotExist(err) {
-		log.Println(logStr, pathname)
-		if err := os.MkdirAll(path.Dir(pathname), 0777); err != nil {
-			return nil, err
-		} else if err := getFile(pathname); err != nil {
-			return nil, err
-		}
-	}
-
-	if digest, err := hex.DecodeString(uf.Digest); err != nil {
-		return nil, err
-	} else {
-		return next(pathname, fromURI, digest)
-	}
-}
-
-func getCloudFile(pathname string, dest string) error {
-	if fd, err := os.Create(dest); err != nil {
-		return err
-	} else {
-		defer fd.Close()
-
-		writer := bufio.NewWriter(fd)
-		if err := getCloudPart(pathname, writer); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func getCloudPart(pathname string, writer *bufio.Writer) error {
-	client := http.Client{}
-	if req, err := http.NewRequest("GET", CloudDAVBase+pathname, nil); err != nil {
-		return err
-	} else {
-		req.SetBasicAuth(CloudUsername, CloudPassword)
-		if resp, err := client.Do(req); err != nil {
-			return err
-		} else {
-			defer resp.Body.Close()
-
-			if resp.StatusCode != http.StatusOK {
-				return errors.New(resp.Status)
-			} else {
-				reader := bufio.NewReader(resp.Body)
-				reader.WriteTo(writer)
-				writer.Flush()
-			}
-		}
-	}
-	return nil
 }

admin/main.go

@@ -13,6 +13,7 @@ import (
 	"text/template"
 
 	"srs.epita.fr/fic-server/admin/api"
+	"srs.epita.fr/fic-server/admin/sync"
 	"srs.epita.fr/fic-server/libfic"
 	"srs.epita.fr/fic-server/settings"
 )
@@ -63,13 +64,13 @@ func StripPrefix(prefix string, h http.Handler) http.Handler {
 func main() {
 	// Read paremeters from environment
 	if v, exists := os.LookupEnv("FICCLOUD_URL"); exists {
-		api.CloudDAVBase = v
+		sync.CloudDAVBase = v
 	}
 	if v, exists := os.LookupEnv("FICCLOUD_USER"); exists {
-		api.CloudUsername = v
+		sync.CloudUsername = v
 	}
 	if v, exists := os.LookupEnv("FICCLOUD_PASS"); exists {
-		api.CloudPassword = v
+		sync.CloudPassword = v
 	}
 
 	// Read parameters from command line
@@ -81,11 +82,10 @@ func main() {
 	flag.StringVar(&api.TeamsDir, "teams", "./TEAMS", "Base directory where save teams JSON files")
 	flag.StringVar(&settings.SettingsDir, "settings", settings.SettingsDir, "Base directory where load and save settings")
 	flag.StringVar(&fic.FilesDir, "files", fic.FilesDir, "Base directory where found challenges files, local part")
-	flag.StringVar(&api.CloudDAVBase, "clouddav", api.CloudDAVBase,
+	flag.StringVar(&sync.CloudDAVBase, "clouddav", sync.CloudDAVBase,
 		"Base directory where found challenges files, cloud part")
-	flag.StringVar(&api.CloudUsername, "clouduser", api.CloudUsername, "Username used to sync")
-	flag.StringVar(&api.CloudPassword, "cloudpass", api.CloudPassword, "Password used to sync")
-	flag.BoolVar(&api.RapidImport, "rapidimport", api.RapidImport, "Don't try to reimport an existing file")
+	flag.StringVar(&sync.CloudUsername, "clouduser", sync.CloudUsername, "Username used to sync")
+	flag.StringVar(&sync.CloudPassword, "cloudpass", sync.CloudPassword, "Password used to sync")
 	flag.BoolVar(&fic.OptionalDigest, "optionaldigest", fic.OptionalDigest, "Is the digest required when importing files?")
 	flag.BoolVar(&fic.StrongDigest, "strongdigest", fic.StrongDigest, "Are BLAKE2b digests required instead of SHA-1 or BLAKE2b?")
 	flag.Parse()

admin/sync/file.go

@@ -0,0 +1,98 @@
+package sync
+
+import (
+	"bufio"
+	"encoding/base32"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"srs.epita.fr/fic-server/libfic"
+
+	"github.com/dchest/blake2b"
+)
+
+type Importer interface {
+	Kind() string
+	exists(filename string) bool
+	toURL(filename string) string
+	getFile(filename string, writer *bufio.Writer) error
+	listDir(filename string) ([]string, error)
+}
+
+var GlobalImporter Importer
+
+func getFile(i Importer, URI string, writer *bufio.Writer) error {
+	// Import file if it exists
+	if i.exists(URI) {
+		return i.getFile(URI, writer)
+	}
+
+	// Try to find file parts
+	dirname := path.Dir(URI)
+	if i.exists(dirname) {
+		filename := path.Base(URI)
+		if files, err := i.listDir(dirname); err != nil {
+			return err
+		} else {
+			for _, fname := range []string{filename, filename + "."} {
+				found := false
+				for _, file := range files {
+					if matched, _ := path.Match(fname + "[0-9][0-9]", file); matched {
+						found = true
+						if err := i.getFile(path.Join(dirname, file), writer); err != nil {
+							return err
+						}
+					}
+				}
+
+				if found {
+					return nil
+				}
+			}
+		}
+	}
+
+	return errors.New(fmt.Sprintf("%q: no such file or directory", i.toURL(URI)))
+}
+
+func ImportFile(URI string, next func(string, string) (interface{}, error)) (interface{}, error) {
+	hash := blake2b.Sum512([]byte(URI))
+	dest := path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), path.Base(URI))
+
+	// If the present file is still valide, don't erase it
+	if _, err := os.Stat(dest); !os.IsNotExist(err) {
+		if r, err := next(dest, URI); err == nil {
+			return r, err
+		}
+
+		if err := os.Remove(dest); err != nil {
+			return nil, err
+		}
+	}
+
+	// Ensure no more file is registered with this path
+	if f, err := fic.GetFileByPath(dest); err == nil {
+		f.Delete()
+	}
+
+	if err := os.MkdirAll(path.Dir(dest), 0755); err != nil {
+		return nil, err
+	}
+
+	// Write file
+	if fdto, err := os.Create(dest); err != nil {
+		return nil, err
+	} else {
+		defer fdto.Close()
+		writer := bufio.NewWriter(fdto)
+		if err := getFile(GlobalImporter, URI, writer); err != nil {
+			os.Remove(dest)
+			return nil, err
+		}
+	}
+
+	return next(dest, URI)
+}

admin/sync/importer_cloud.go

@@ -0,0 +1,96 @@
+package sync
+
+import (
+	"bufio"
+	"errors"
+	"net/http"
+	"net/url"
+	"path"
+
+	"github.com/studio-b12/gowebdav"
+)
+
+var CloudDAVBase string = "https://srs.epita.fr/owncloud/remote.php/webdav/FIC 2018"
+var CloudUsername string = "fic"
+var CloudPassword string = ""
+
+type CloudImporter struct {
+	baseDAV url.URL
+	username string
+	password string
+}
+
+func NewCloudImporter(baseDAV string, username string, password string) (*CloudImporter, error) {
+	if r, err := url.Parse(baseDAV); err != nil {
+		return nil, err
+	} else {
+		return &CloudImporter{*r, username, password}, nil
+	}
+}
+
+func (i CloudImporter) Kind() string {
+	return "cloud file importer: " + i.baseDAV.String()
+}
+
+func (i CloudImporter) exists(filename string) bool {
+	fullURL := i.baseDAV
+	fullURL.Path = path.Join(fullURL.Path, filename)
+
+	client := http.Client{}
+	if req, err := http.NewRequest("HEAD", fullURL.String(), nil); err == nil {
+		req.SetBasicAuth(i.username, i.password)
+
+		if resp, err := client.Do(req); err == nil {
+			defer resp.Body.Close()
+
+			return resp.StatusCode == http.StatusOK
+		}
+	}
+	return false
+}
+
+func (i CloudImporter) toURL(filename string) string {
+	fullURL := i.baseDAV
+	fullURL.Path = path.Join(fullURL.Path, filename)
+	return fullURL.String()
+}
+
+func (i CloudImporter) getFile(filename string, writer *bufio.Writer) error {
+	fullURL := i.baseDAV
+	fullURL.Path = path.Join(fullURL.Path, filename)
+
+	client := http.Client{}
+	if req, err := http.NewRequest("GET", fullURL.String(), nil); err != nil {
+		return err
+	} else {
+		req.SetBasicAuth(i.username, i.password)
+		if resp, err := client.Do(req); err != nil {
+			return err
+		} else {
+			defer resp.Body.Close()
+
+			if resp.StatusCode != http.StatusOK {
+				return errors.New(resp.Status)
+			} else {
+				reader := bufio.NewReader(resp.Body)
+				reader.WriteTo(writer)
+				writer.Flush()
+			}
+		}
+	}
+	return nil
+}
+
+func (i CloudImporter) listDir(filename string) ([]string, error) {
+	client := gowebdav.NewClient(i.baseDAV.String(), i.username, i.password)
+
+	if files, err := client.ReadDir(filename); err != nil {
+		return nil, err
+	} else {
+		res := make([]string, 0)
+		for _, file := range files {
+			res = append(res, file.Name())
+		}
+		return res, nil
+	}
+}

admin/sync/importer_localfs.go

@@ -0,0 +1,49 @@
+package sync
+
+import (
+	"bufio"
+	"io/ioutil"
+	"os"
+	"path"
+)
+
+type LocalImporter struct {
+	Base string
+}
+
+func (i LocalImporter) Kind() string {
+	return "local file importer: " + i.Base
+}
+
+func (i LocalImporter) exists(filename string) bool {
+	_, err := os.Stat(path.Join(i.Base, filename))
+	return !os.IsNotExist(err)
+}
+
+func (i LocalImporter) toURL(filename string) string {
+	return path.Join(i.Base, filename)
+}
+
+func (i LocalImporter) getFile(filename string, writer *bufio.Writer) error {
+	if fd, err := os.Open(path.Join(i.Base, filename)); err != nil {
+		return err
+	} else {
+		defer fd.Close()
+		reader := bufio.NewReader(fd)
+		reader.WriteTo(writer)
+		writer.Flush()
+		return nil
+	}
+}
+
+func (i LocalImporter) listDir(filename string) ([]string, error) {
+	if files, err := ioutil.ReadDir(path.Join(i.Base, filename)); err != nil {
+		return nil, err
+	} else {
+		res := make([]string, 0)
+		for _, file := range files {
+			res = append(res, file.Name())
+		}
+		return res, nil
+	}
+}

libfic/file.go

@@ -135,7 +135,23 @@ func (e Exercice) ImportFile(filePath string, origin string, digest []byte) (int
 			}
 		}
 
-		return e.AddFile(strings.TrimPrefix(filePath, FilesDir), origin, path.Base(filePath), result512, fi.Size())
+		dPath := strings.TrimPrefix(filePath, FilesDir)
+		if f, err := e.GetFileByPath(dPath); err != nil {
+			return e.AddFile(dPath, origin, path.Base(filePath), result512, fi.Size())
+		} else {
+			// Don't need to update Path and Name, because they are related to dPath
+
+			f.IdExercice = e.Id
+			f.origin = origin
+			f.Checksum = result512
+			f.Size = fi.Size()
+
+			if _, err := f.Update(); err != nil {
+				return nil, err
+			} else {
+				return f, nil
+			}
+		}
 	}
 }