server/admin/sync/file.go

package sync

import (
	"bufio"
	"bytes"
	"encoding/base32"
	"fmt"
	"io"
	"net/url"
	"os"
	"path"
	"strings"

	"srs.epita.fr/fic-server/libfic"

	"golang.org/x/crypto/blake2b"
)

// Importer are abstract methods required to import challenges.
type Importer interface {
	// Kind returns information about the Importer, for human interrest.
	Kind() string
	// Id returns information about the current state (commit id, ...).
	Id() *string
	// init performs the importer initialization.
	Init() error
	// sync tries to pull the latest modification of the underlying storage.
	Sync() error
	// Exists checks if the given location exists from the Importer point of view.
	Exists(filename string) bool
	// toURL gets the full path/URL to the given file, the Importer will look internaly (used for debuging purpose).
	toURL(filename string) string
	// importFile imports the file at the given URI, inside the global FILES/ directory.
	// Then calls back the next function, with the downloaded location and the original URI.
	// Callback return is forwarded.
	importFile(URI string, next func(string, string) (interface{}, error)) (interface{}, error)
	// getFileReader returns a reader to the requested file.
	GetFile(filename string) (io.Reader, error)
	// listDir returns a list of the files and subdirectories contained inside the directory at the given location.
	ListDir(filename string) ([]string, error)
	// stat returns many information about the given file: such as last modification date, size, ...
	Stat(filename string) (os.FileInfo, error)
}

// DirectAccessImporter abstracts importer that support direct file access through a local path
type DirectAccessImporter interface {
	GetLocalPath(p ...string) string
}

// ForgeLinkedImporter abstracts importer that are linked to a forge
type ForgeLinkedImporter interface {
	GetThemeLink(th *fic.Theme) (*url.URL, error)
	GetExerciceLink(e *fic.Exercice) (*url.URL, error)
}

// WritableImporter abstracts importer that we can also write on
type WritableImporter interface {
	// writeFile write the given buffer to the file at the given location.
	writeFile(filename string, reader io.Reader) error
}

// GlobalImporter stores the main importer instance to use for global imports.
var GlobalImporter Importer

// GetFileSize returns the size.
func GetFileSize(i Importer, URI string) (size int64, err error) {
	if i.Exists(URI) {
		if fi, err := i.Stat(URI); err != nil {
			return 0, err
		} else {
			return fi.Size(), nil
		}
	}

	dirname := path.Dir(URI)
	if i.Exists(dirname) {
		filename := path.Base(URI)
		if files, err := i.ListDir(dirname); err != nil {
			return size, err
		} else {
			for _, fname := range []string{filename, filename + "."} {
				found := false
				for _, file := range files {
					if matched, _ := path.Match(fname+"[0-9][0-9]", file); matched {
						found = true
						if fi, err := i.Stat(path.Join(dirname, file)); err != nil {
							return size, err
						} else {
							size += fi.Size()
						}
					}
				}

				if found {
					return size, nil
				}
			}
		}
	}

	return size, fmt.Errorf("%q: no such file or directory", URI)
}

// GetFile helps to manage huge file transfert by concatenating splitted (with split(1)) files.
func GetFile(i Importer, URI string) (io.Reader, func(), error) {
	// Import file if it exists
	if i.Exists(URI) {
		fd, err := i.GetFile(URI)
		return fd, func() {
			if fdc, ok := fd.(io.ReadCloser); ok {
				fdc.Close()
			}
		}, err
	}

	// Try to find file parts
	dirname := path.Dir(URI)
	if i.Exists(dirname) {
		filename := path.Base(URI)
		if files, err := i.ListDir(dirname); err != nil {
			return nil, nil, err
		} else {
			var readers []io.Reader

			for _, fname := range []string{filename, filename + "."} {
				for _, file := range files {
					if matched, _ := path.Match(fname+"[0-9][0-9]", file); matched {
						fd, err := i.GetFile(path.Join(dirname, file))
						if err != nil {
							// Close already opened files to avoid leaks
							for _, rd := range readers {
								if rdc, ok := rd.(io.ReadCloser); ok {
									rdc.Close()
								}
							}

							return nil, nil, err
						}

						readers = append(readers, fd)
					}
				}

				if len(readers) > 0 {
					return io.MultiReader(readers...), func() {
						for _, rd := range readers {
							if rdc, ok := rd.(io.ReadCloser); ok {
								rdc.Close()
							}
						}
					}, nil
				}
			}
		}
	}

	return nil, nil, fmt.Errorf("%q: no such file or directory", URI)
}

// GetFileContent retrieves the content of the given text file.
func GetFileContent(i Importer, URI string) (string, error) {
	if fd, closer, err := GetFile(i, URI); err != nil {
		return "", err
	} else {
		defer closer()

		buffd := bufio.NewReader(fd)

		// Ensure we read UTF-8 content.
		buf := make([]rune, 0)
		for b, _, err := buffd.ReadRune(); err == nil; b, _, err = buffd.ReadRune() {
			buf = append(buf, b)
		}

		return strings.TrimSpace(string(buf)), nil
	}
}

// GetDestinationFilePath generates the destination path, from the URI.
// This function permits to obfusce to player the original URI.
// Theoricaly, changing the import method doesn't change destination URI.
func GetDestinationFilePath(URI string, filename *string) string {
	if filename == nil {
		tmp := path.Base(URI)
		filename = &tmp
	}
	hash := blake2b.Sum512([]byte(URI))
	return path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), *filename)
}

func importFile(i Importer, URI string, dest string) error {
	if err := os.MkdirAll(path.Dir(dest), 0751); err != nil {
		return err
	}

	// Write file
	if fdto, err := os.Create(dest); err != nil {
		return err
	} else {
		defer fdto.Close()

		if fdfrom, closer, err := GetFile(i, URI); err != nil {
			os.Remove(dest)
			return err
		} else {
			defer closer()

			_, err = io.Copy(fdto, fdfrom)
			return err
		}
	}
}

// ImportFile imports the file at the given URI, using helpers of the given Importer.
// After import, next is called with relative path where the file has been saved and the original URI.
func ImportFile(i Importer, URI string, next func(string, string) (interface{}, error)) (interface{}, error) {
	dest := GetDestinationFilePath(URI, nil)

	// If the present file is still valide, don't erase it
	if _, err := os.Stat(dest); !os.IsNotExist(err) {
		if r, err := next(dest, URI); err == nil {
			return r, err
		}
	}

	if err := importFile(i, URI, dest); err != nil {
		return nil, err
	}

	return next(dest, URI)
}

// WriteFileContent save the given content to the given text file.
func WriteFileContent(i Importer, URI string, content []byte) error {
	if wi, ok := i.(WritableImporter); ok {
		return wi.writeFile(URI, bytes.NewReader(content))
	} else {
		return fmt.Errorf("%t is not capable of writing", i)
	}
}