Bufferize file download (to avoid OOM)

admin/api_file.go

@@ -1,11 +1,11 @@
 package main
 
 import (
+	"bufio"
 	"crypto/sha512"
 	"encoding/base32"
 	"encoding/json"
 	"errors"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -34,39 +34,42 @@ func createExerciceFile(theme fic.Theme, exercice fic.Exercice, args []string, b
 
 	if _, err := os.Stat(pathname); os.IsNotExist(err) {
 		log.Println("Import file from Cloud:", uf.URI, "=>", pathname)
-		if body, err := getCloudFile(uf.URI); err != nil {
+		if err := os.MkdirAll(path.Dir(pathname), 0777); err != nil {
+			return nil, err
+		} else if err := getCloudFile(uf.URI, pathname); err != nil {
 			return nil, err
-		} else {
-			if err := os.MkdirAll(path.Dir(pathname), 0777); err != nil {
-				return nil, err
-			}
-
-			if err := ioutil.WriteFile(pathname, body, 0777); err != nil {
-				return nil, err
-			}
 		}
 	}
 
 	return exercice.ImportFile(pathname, uf.URI)
 }
 
-func getCloudFile(pathname string) ([]byte, error) {
+func getCloudFile(pathname string, dest string) (error) {
 	client := http.Client{}
 	if req, err := http.NewRequest("GET", CloudDAVBase+pathname, nil); err != nil {
-		return nil, err
+		return err
 	} else {
 		req.SetBasicAuth(CloudUsername, CloudPassword)
 		if resp, err := client.Do(req); err != nil {
-			return nil, err
+			return err
 		} else {
 			defer resp.Body.Close()
-			if resp.StatusCode != http.StatusOK {
-				return nil, errors.New(resp.Status)
-			} else if body, err := ioutil.ReadAll(resp.Body); err != nil {
-				return nil, err
+
+			if fd, err := os.Create(dest); err != nil {
+				return err
 			} else {
-				return body, nil
+				defer fd.Close();
+
+				if resp.StatusCode != http.StatusOK {
+					return errors.New(resp.Status)
+				} else {
+					writer := bufio.NewWriter(fd)
+					reader := bufio.NewReader(resp.Body)
+					reader.WriteTo(writer)
+					writer.Flush()
+				}
 			}
 		}
 	}
+	return nil
 }

libfic/file.go

@@ -1,6 +1,7 @@
 package fic
 
 import (
+	"bufio"
 	"crypto/sha1"
 	"io"
 	"os"
@@ -11,7 +12,7 @@ import (
 var FilesDir string
 
 type EFile struct {
-	Id         int64  `json:"id"`
+	Id         int64 `json:"id"`
 	origin     string
 	Path       string `json:"path"`
 	IdExercice int64  `json:"idExercice"`
@@ -51,8 +52,9 @@ func (e Exercice) ImportFile(filePath string, origin string) (EFile, error) {
 	} else {
 		defer fd.Close()
 
+		reader := bufio.NewReader(fd)
 		hash := sha1.New()
-		if _, err := io.Copy(hash, fd); err != nil {
+		if _, err := io.Copy(hash, reader); err != nil {
 			return EFile{}, err
 		}