diff --git a/admin/api/exercice.go b/admin/api/exercice.go index 2fd06c57..3e96c4b0 100644 --- a/admin/api/exercice.go +++ b/admin/api/exercice.go @@ -2,10 +2,12 @@ package api import ( "encoding/json" + "encoding/hex" "errors" "strings" "srs.epita.fr/fic-server/libfic" + "srs.epita.fr/fic-server/admin/sync" "github.com/julienschmidt/httprouter" ) @@ -125,7 +127,6 @@ type uploadedHint struct { Content string Cost int64 URI string - Path string } func createExerciceHint(exercice fic.Exercice, body []byte) (interface{}, error) { @@ -136,9 +137,9 @@ func createExerciceHint(exercice fic.Exercice, body []byte) (interface{}, error) if len(uh.Content) != 0 { return exercice.AddHint(uh.Title, uh.Content, uh.Cost) - } else if len(uh.Path) != 0 || len(uh.URI) != 0 { - return importFile(uploadedFile{Path: uh.Path, URI: uh.URI}, - func(filePath string, origin string, digest []byte) (interface{}, error) { + } else if len(uh.URI) != 0 { + return sync.ImportFile(uh.URI, + func(filePath string, origin string) (interface{}, error) { return exercice.AddHint(uh.Title, "$FILES" + strings.TrimPrefix(filePath, fic.FilesDir), uh.Cost) }) } else { @@ -209,7 +210,14 @@ func createExerciceFile(exercice fic.Exercice, body []byte) (interface{}, error) return nil, err } - return importFile(uf, exercice.ImportFile) + return sync.ImportFile(uf.URI, + func(filePath string, origin string) (interface{}, error) { + if digest, err := hex.DecodeString(uf.Digest); err != nil { + return nil, err + } else { + return exercice.ImportFile(filePath, origin, digest) + } + }) } func showExerciceFile(file fic.EFile, body []byte) (interface{}, error) { diff --git a/admin/api/file.go b/admin/api/file.go index a496115a..e2179371 100644 --- a/admin/api/file.go +++ b/admin/api/file.go @@ -1,154 +1,8 @@ package api -import ( - "bufio" - "encoding/base32" - "encoding/hex" - "errors" - "fmt" - "log" - "net/http" - "os" - "path" - "strings" - - "srs.epita.fr/fic-server/libfic" - - "github.com/dchest/blake2b" -) - -var CloudDAVBase string = "https://srs.epita.fr/owncloud/remote.php/webdav/FIC 2018" -var CloudUsername string = "fic" -var CloudPassword string = "" -var RapidImport bool = false +import () type uploadedFile struct { URI string Digest string - Path string - Parts []string -} - -func importFile(uf uploadedFile, next func(string, string, []byte) (interface{}, error)) (interface{}, error) { - var hash [blake2b.Size]byte - var logStr string - var fromURI string - var getFile func(string) error - - if uf.URI != "" && len(uf.Parts) > 0 { - hash = blake2b.Sum512([]byte(uf.URI)) - logStr = fmt.Sprintf("Import file from Cloud: %s =>", uf.Parts) - fromURI = uf.URI - getFile = func(dest string) error { - if fdto, err := os.Create(dest); err != nil { - return err - } else { - writer := bufio.NewWriter(fdto) - for _, partname := range uf.Parts { - if err := getCloudPart(partname, writer); err != nil { - return err - } - } - fdto.Close() - } - return nil - } - } else if uf.URI != "" { - hash = blake2b.Sum512([]byte(uf.URI)) - logStr = "Import file from Cloud: " + uf.URI + " =>" - fromURI = uf.URI - getFile = func(dest string) error { return getCloudFile(uf.URI, dest) } - } else if uf.Path != "" && len(uf.Parts) > 0 { - hash = blake2b.Sum512([]byte(uf.Path)) - logStr = fmt.Sprintf("Import file from local FS: %s =>", uf.Parts) - fromURI = uf.Path - getFile = func(dest string) error { - if fdto, err := os.Create(dest); err != nil { - return err - } else { - writer := bufio.NewWriter(fdto) - for _, partname := range uf.Parts { - if fdfrm, err := os.Open(partname); err != nil { - return err - } else { - reader := bufio.NewReader(fdfrm) - reader.WriteTo(writer) - writer.Flush() - fdfrm.Close() - } - } - fdto.Close() - } - return nil - } - } else if uf.Path != "" { - hash = blake2b.Sum512([]byte(uf.Path)) - logStr = "Import file from local FS: " + uf.Path + " =>" - fromURI = uf.Path - getFile = func(dest string) error { return os.Symlink(uf.Path, dest) } - } else { - return nil, errors.New("URI or path not filled") - } - - pathname := path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(0).EncodeToString(hash[:])), path.Base(fromURI)) - - // Remove the file if it exists - // TODO: check if this is symlink => remove to avoid File not found error after, because the file is writen at the adresse pointed. - if _, err := os.Stat(pathname); !os.IsNotExist(err) && !RapidImport { - if err := os.Remove(pathname); err != nil { - return nil, err - } - } - - if _, err := os.Stat(pathname); os.IsNotExist(err) { - log.Println(logStr, pathname) - if err := os.MkdirAll(path.Dir(pathname), 0777); err != nil { - return nil, err - } else if err := getFile(pathname); err != nil { - return nil, err - } - } - - if digest, err := hex.DecodeString(uf.Digest); err != nil { - return nil, err - } else { - return next(pathname, fromURI, digest) - } -} - -func getCloudFile(pathname string, dest string) error { - if fd, err := os.Create(dest); err != nil { - return err - } else { - defer fd.Close() - - writer := bufio.NewWriter(fd) - if err := getCloudPart(pathname, writer); err != nil { - return err - } - } - return nil -} - -func getCloudPart(pathname string, writer *bufio.Writer) error { - client := http.Client{} - if req, err := http.NewRequest("GET", CloudDAVBase+pathname, nil); err != nil { - return err - } else { - req.SetBasicAuth(CloudUsername, CloudPassword) - if resp, err := client.Do(req); err != nil { - return err - } else { - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return errors.New(resp.Status) - } else { - reader := bufio.NewReader(resp.Body) - reader.WriteTo(writer) - writer.Flush() - } - } - } - return nil } diff --git a/admin/main.go b/admin/main.go index 58f2bba3..09af45b5 100644 --- a/admin/main.go +++ b/admin/main.go @@ -13,6 +13,7 @@ import ( "text/template" "srs.epita.fr/fic-server/admin/api" + "srs.epita.fr/fic-server/admin/sync" "srs.epita.fr/fic-server/libfic" "srs.epita.fr/fic-server/settings" ) @@ -63,13 +64,13 @@ func StripPrefix(prefix string, h http.Handler) http.Handler { func main() { // Read paremeters from environment if v, exists := os.LookupEnv("FICCLOUD_URL"); exists { - api.CloudDAVBase = v + sync.CloudDAVBase = v } if v, exists := os.LookupEnv("FICCLOUD_USER"); exists { - api.CloudUsername = v + sync.CloudUsername = v } if v, exists := os.LookupEnv("FICCLOUD_PASS"); exists { - api.CloudPassword = v + sync.CloudPassword = v } // Read parameters from command line @@ -81,11 +82,10 @@ func main() { flag.StringVar(&api.TeamsDir, "teams", "./TEAMS", "Base directory where save teams JSON files") flag.StringVar(&settings.SettingsDir, "settings", settings.SettingsDir, "Base directory where load and save settings") flag.StringVar(&fic.FilesDir, "files", fic.FilesDir, "Base directory where found challenges files, local part") - flag.StringVar(&api.CloudDAVBase, "clouddav", api.CloudDAVBase, + flag.StringVar(&sync.CloudDAVBase, "clouddav", sync.CloudDAVBase, "Base directory where found challenges files, cloud part") - flag.StringVar(&api.CloudUsername, "clouduser", api.CloudUsername, "Username used to sync") - flag.StringVar(&api.CloudPassword, "cloudpass", api.CloudPassword, "Password used to sync") - flag.BoolVar(&api.RapidImport, "rapidimport", api.RapidImport, "Don't try to reimport an existing file") + flag.StringVar(&sync.CloudUsername, "clouduser", sync.CloudUsername, "Username used to sync") + flag.StringVar(&sync.CloudPassword, "cloudpass", sync.CloudPassword, "Password used to sync") flag.BoolVar(&fic.OptionalDigest, "optionaldigest", fic.OptionalDigest, "Is the digest required when importing files?") flag.BoolVar(&fic.StrongDigest, "strongdigest", fic.StrongDigest, "Are BLAKE2b digests required instead of SHA-1 or BLAKE2b?") flag.Parse() diff --git a/admin/sync/file.go b/admin/sync/file.go new file mode 100644 index 00000000..dee147c7 --- /dev/null +++ b/admin/sync/file.go @@ -0,0 +1,98 @@ +package sync + +import ( + "bufio" + "encoding/base32" + "errors" + "fmt" + "os" + "path" + "strings" + + "srs.epita.fr/fic-server/libfic" + + "github.com/dchest/blake2b" +) + +type Importer interface { + Kind() string + exists(filename string) bool + toURL(filename string) string + getFile(filename string, writer *bufio.Writer) error + listDir(filename string) ([]string, error) +} + +var GlobalImporter Importer + +func getFile(i Importer, URI string, writer *bufio.Writer) error { + // Import file if it exists + if i.exists(URI) { + return i.getFile(URI, writer) + } + + // Try to find file parts + dirname := path.Dir(URI) + if i.exists(dirname) { + filename := path.Base(URI) + if files, err := i.listDir(dirname); err != nil { + return err + } else { + for _, fname := range []string{filename, filename + "."} { + found := false + for _, file := range files { + if matched, _ := path.Match(fname + "[0-9][0-9]", file); matched { + found = true + if err := i.getFile(path.Join(dirname, file), writer); err != nil { + return err + } + } + } + + if found { + return nil + } + } + } + } + + return errors.New(fmt.Sprintf("%q: no such file or directory", i.toURL(URI))) +} + +func ImportFile(URI string, next func(string, string) (interface{}, error)) (interface{}, error) { + hash := blake2b.Sum512([]byte(URI)) + dest := path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), path.Base(URI)) + + // If the present file is still valide, don't erase it + if _, err := os.Stat(dest); !os.IsNotExist(err) { + if r, err := next(dest, URI); err == nil { + return r, err + } + + if err := os.Remove(dest); err != nil { + return nil, err + } + } + + // Ensure no more file is registered with this path + if f, err := fic.GetFileByPath(dest); err == nil { + f.Delete() + } + + if err := os.MkdirAll(path.Dir(dest), 0755); err != nil { + return nil, err + } + + // Write file + if fdto, err := os.Create(dest); err != nil { + return nil, err + } else { + defer fdto.Close() + writer := bufio.NewWriter(fdto) + if err := getFile(GlobalImporter, URI, writer); err != nil { + os.Remove(dest) + return nil, err + } + } + + return next(dest, URI) +} diff --git a/admin/sync/importer_cloud.go b/admin/sync/importer_cloud.go new file mode 100644 index 00000000..027147ce --- /dev/null +++ b/admin/sync/importer_cloud.go @@ -0,0 +1,96 @@ +package sync + +import ( + "bufio" + "errors" + "net/http" + "net/url" + "path" + + "github.com/studio-b12/gowebdav" +) + +var CloudDAVBase string = "https://srs.epita.fr/owncloud/remote.php/webdav/FIC 2018" +var CloudUsername string = "fic" +var CloudPassword string = "" + +type CloudImporter struct { + baseDAV url.URL + username string + password string +} + +func NewCloudImporter(baseDAV string, username string, password string) (*CloudImporter, error) { + if r, err := url.Parse(baseDAV); err != nil { + return nil, err + } else { + return &CloudImporter{*r, username, password}, nil + } +} + +func (i CloudImporter) Kind() string { + return "cloud file importer: " + i.baseDAV.String() +} + +func (i CloudImporter) exists(filename string) bool { + fullURL := i.baseDAV + fullURL.Path = path.Join(fullURL.Path, filename) + + client := http.Client{} + if req, err := http.NewRequest("HEAD", fullURL.String(), nil); err == nil { + req.SetBasicAuth(i.username, i.password) + + if resp, err := client.Do(req); err == nil { + defer resp.Body.Close() + + return resp.StatusCode == http.StatusOK + } + } + return false +} + +func (i CloudImporter) toURL(filename string) string { + fullURL := i.baseDAV + fullURL.Path = path.Join(fullURL.Path, filename) + return fullURL.String() +} + +func (i CloudImporter) getFile(filename string, writer *bufio.Writer) error { + fullURL := i.baseDAV + fullURL.Path = path.Join(fullURL.Path, filename) + + client := http.Client{} + if req, err := http.NewRequest("GET", fullURL.String(), nil); err != nil { + return err + } else { + req.SetBasicAuth(i.username, i.password) + if resp, err := client.Do(req); err != nil { + return err + } else { + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return errors.New(resp.Status) + } else { + reader := bufio.NewReader(resp.Body) + reader.WriteTo(writer) + writer.Flush() + } + } + } + return nil +} + +func (i CloudImporter) listDir(filename string) ([]string, error) { + client := gowebdav.NewClient(i.baseDAV.String(), i.username, i.password) + + if files, err := client.ReadDir(filename); err != nil { + return nil, err + } else { + res := make([]string, 0) + for _, file := range files { + res = append(res, file.Name()) + } + return res, nil + } +} diff --git a/admin/sync/importer_localfs.go b/admin/sync/importer_localfs.go new file mode 100644 index 00000000..a281a8ee --- /dev/null +++ b/admin/sync/importer_localfs.go @@ -0,0 +1,49 @@ +package sync + +import ( + "bufio" + "io/ioutil" + "os" + "path" +) + +type LocalImporter struct { + Base string +} + +func (i LocalImporter) Kind() string { + return "local file importer: " + i.Base +} + +func (i LocalImporter) exists(filename string) bool { + _, err := os.Stat(path.Join(i.Base, filename)) + return !os.IsNotExist(err) +} + +func (i LocalImporter) toURL(filename string) string { + return path.Join(i.Base, filename) +} + +func (i LocalImporter) getFile(filename string, writer *bufio.Writer) error { + if fd, err := os.Open(path.Join(i.Base, filename)); err != nil { + return err + } else { + defer fd.Close() + reader := bufio.NewReader(fd) + reader.WriteTo(writer) + writer.Flush() + return nil + } +} + +func (i LocalImporter) listDir(filename string) ([]string, error) { + if files, err := ioutil.ReadDir(path.Join(i.Base, filename)); err != nil { + return nil, err + } else { + res := make([]string, 0) + for _, file := range files { + res = append(res, file.Name()) + } + return res, nil + } +} diff --git a/libfic/file.go b/libfic/file.go index 163b3119..c887509a 100644 --- a/libfic/file.go +++ b/libfic/file.go @@ -135,7 +135,23 @@ func (e Exercice) ImportFile(filePath string, origin string, digest []byte) (int } } - return e.AddFile(strings.TrimPrefix(filePath, FilesDir), origin, path.Base(filePath), result512, fi.Size()) + dPath := strings.TrimPrefix(filePath, FilesDir) + if f, err := e.GetFileByPath(dPath); err != nil { + return e.AddFile(dPath, origin, path.Base(filePath), result512, fi.Size()) + } else { + // Don't need to update Path and Name, because they are related to dPath + + f.IdExercice = e.Id + f.origin = origin + f.Checksum = result512 + f.Size = fi.Size() + + if _, err := f.Update(); err != nil { + return nil, err + } else { + return f, nil + } + } } }