Refactor sync file reading

admin/sync/file.go

@@ -102,21 +102,15 @@ func getFileSize(i Importer, URI string) (size int64, err error) {
 }
 
 // GetFile helps to manage huge file transfert by concatenating splitted (with split(1)) files.
-func GetFile(i Importer, URI string, writer *bufio.Writer) error {
+func GetFile(i Importer, URI string) (io.Reader, func(), error) {
 	// Import file if it exists
 	if i.exists(URI) {
 		fd, err := i.getFile(URI)
-		if err != nil {
-			return err
-		}
-
-		if fdc, ok := fd.(io.ReadCloser); ok {
-			defer fdc.Close()
-		}
-		writer.ReadFrom(fd)
-
-		writer.Flush()
-		return nil
+		return fd, func() {
+			if fdc, ok := fd.(io.ReadCloser); ok {
+				fdc.Close()
+			}
+		}, err
 	}
 
 	// Try to find file parts
@@ -124,47 +118,57 @@ func GetFile(i Importer, URI string, writer *bufio.Writer) error {
 	if i.exists(dirname) {
 		filename := path.Base(URI)
 		if files, err := i.listDir(dirname); err != nil {
-			return err
+			return nil, nil, err
 		} else {
+			var readers []io.Reader
+
 			for _, fname := range []string{filename, filename + "."} {
-				found := false
 				for _, file := range files {
 					if matched, _ := path.Match(fname+"[0-9][0-9]", file); matched {
-						found = true
-
 						fd, err := i.getFile(path.Join(dirname, file))
 						if err != nil {
-							return err
+							// Close already opened files to avoid leaks
+							for _, rd := range readers {
+								if rdc, ok := rd.(io.ReadCloser); ok {
+									rdc.Close()
+								}
+							}
+
+							return nil, nil, err
 						}
 
-						if fdc, ok := fd.(io.ReadCloser); ok {
-							defer fdc.Close()
-						}
-						writer.ReadFrom(fd)
+						readers = append(readers, fd)
 					}
 				}
 
-				if found {
-					writer.Flush()
-					return nil
+				if len(readers) > 0 {
+					return io.MultiReader(readers...), func() {
+						for _, rd := range readers {
+							if rdc, ok := rd.(io.ReadCloser); ok {
+								rdc.Close()
+							}
+						}
+					}, nil
 				}
 			}
 		}
 	}
 
-	return fmt.Errorf("%q: no such file or directory", URI)
+	return nil, nil, fmt.Errorf("%q: no such file or directory", URI)
 }
 
 // GetFileContent retrieves the content of the given text file.
 func GetFileContent(i Importer, URI string) (string, error) {
-	cnt := bytes.Buffer{}
-
-	if err := GetFile(i, URI, bufio.NewWriter(io.Writer(&cnt))); err != nil {
+	if fd, closer, err := GetFile(i, URI); err != nil {
 		return "", err
 	} else {
+		defer closer()
+
+		buffd := bufio.NewReader(fd)
+
 		// Ensure we read UTF-8 content.
 		buf := make([]rune, 0)
-		for b, _, err := cnt.ReadRune(); err == nil; b, _, err = cnt.ReadRune() {
+		for b, _, err := buffd.ReadRune(); err == nil; b, _, err = buffd.ReadRune() {
 			buf = append(buf, b)
 		}
 
@@ -194,14 +198,17 @@ func importFile(i Importer, URI string, dest string) error {
 		return err
 	} else {
 		defer fdto.Close()
-		writer := bufio.NewWriter(fdto)
-		if err := GetFile(i, URI, writer); err != nil {
+
+		if fdfrom, closer, err := GetFile(i, URI); err != nil {
 			os.Remove(dest)
 			return err
+		} else {
+			defer closer()
+
+			_, err = io.Copy(fdto, fdfrom)
+			return err
 		}
 	}
-
-	return nil
 }
 
 // ImportFile imports the file at the given URI, using helpers of the given Importer.