From a4c87b92a52e95ab245573dcaea65032edd81d2c Mon Sep 17 00:00:00 2001
From: Pierre-Olivier Mercier <nemunaire@nemunai.re>
Date: Mon, 20 Jan 2020 08:41:36 +0100
Subject: [PATCH] admin: introducing speedy deep sync and themed deep sync

---
 admin/api/theme.go               |  13 +++
 admin/static/check_import.html   |   4 +-
 admin/static/js/app.js           |  26 +++++-
 admin/static/views/settings.html |  11 ++-
 admin/sync/full.go               | 147 +++++++++++++++++++++++++------
 5 files changed, 170 insertions(+), 31 deletions(-)

diff --git a/admin/api/theme.go b/admin/api/theme.go
index 3caf2e98..97832c77 100644
--- a/admin/api/theme.go
+++ b/admin/api/theme.go
@@ -52,10 +52,23 @@ func init() {
 				return map[string]interface{}{"progress": sync.DeepSyncProgress}, nil
 			}
 		}))
+	router.POST("/api/sync/speed", apiHandler(
+		func(_ httprouter.Params, _ []byte) (interface{}, error) {
+			st := sync.SpeedySyncDeep(sync.GlobalImporter)
+			sync.EditDeepReport(st, false)
+			return st, nil
+		}))
 	router.POST("/api/sync/deep", apiHandler(
 		func(_ httprouter.Params, _ []byte) (interface{}, error) {
 			return sync.SyncDeep(sync.GlobalImporter), nil
 		}))
+	router.POST("/api/sync/deep/:thid", apiHandler(themeHandler(
+		func(theme fic.Theme, _ []byte) (interface{}, error) {
+			st := sync.SyncThemeDeep(sync.GlobalImporter, theme, 0, 250)
+			sync.EditDeepReport(map[string][]string{theme.Name: st}, false)
+			sync.DeepSyncProgress = 255
+			return st, nil
+		})))
 	router.POST("/api/sync/themes", apiHandler(
 		func(_ httprouter.Params, _ []byte) (interface{}, error) {
 			return sync.SyncThemes(sync.GlobalImporter), nil
diff --git a/admin/static/check_import.html b/admin/static/check_import.html
index 25787ade..f6720e61 100644
--- a/admin/static/check_import.html
+++ b/admin/static/check_import.html
@@ -8,8 +8,8 @@
     </style>
     <script type="text/javascript">
       function disp(data) {
-      if (data["_date"] && data["_date"].length == 2) {
-          document.getElementById("date_imp").innerHTML = data["_date"][1];
+      if (data["_date"]) {
+          document.getElementById("date_imp").innerHTML = data["_date"][data["_date"].length - 1];
       }
       Object.keys(data).map(function(theme) {
         if (data[theme] != null && theme != "_date") {
diff --git a/admin/static/js/app.js b/admin/static/js/app.js
index 4e19079e..48f7f2e5 100644
--- a/admin/static/js/app.js
+++ b/admin/static/js/app.js
@@ -554,11 +554,18 @@ angular.module("FICApp")
 	                           });
         };
 	$scope.deepSyncInProgress = false;
-	$scope.deepSync = function(type) {
-	    $rootScope.newYesNoBox('warning', 'Faire une synchronisation intégrale ?', '',
+	$scope.deepSync = function(theme) {
+	  if (theme) {
+	    question = 'Faire une synchronisation intégrale du thème ' + theme.name + ' ?'
+	    url = "/api/sync/deep/" + theme.id
+	  } else {
+	    question = 'Faire une synchronisation intégrale ?'
+	    url = "/api/sync/deep"
+	  }
+	  $rootScope.newYesNoBox('warning', question, '',
 				   function() {
 				       $scope.deepSyncInProgress = true;
-				       $http.post("/api/sync/deep").then(function() {
+				       $http.post(url).then(function() {
 					   $scope.deepSyncInProgress = false;
 					   $rootScope.newBox('success', 'Synchronisation intégrale terminée.', '<a href="check_import.html">Voir le rapport</a>.', 15000);
 				       }, function(response) {
@@ -567,6 +574,19 @@ angular.module("FICApp")
 				       });
 				   });
 	};
+	$scope.speedyDeepSync = function() {
+	    $rootScope.newYesNoBox('warning', 'Faire une synchronisation profonde rapide, sans s\'occuper des fichiers ?', '',
+				   function() {
+				       $scope.deepSyncInProgress = true;
+				       $http.post("/api/sync/speed").then(function() {
+					   $scope.deepSyncInProgress = false;
+					   $rootScope.newBox('success', 'Synchronisation profonde rapide terminée.', '<a href="check_import.html">Voir le rapport</a>.', 15000);
+				       }, function(response) {
+					   $scope.deepSyncInProgress = false;
+					   $rootScope.newBox('warning', 'Synchronisation profinde rapide terminée.', '<a href="check_import.html">Voir le rapport</a>.', 15000);
+				       });
+				   });
+	};
     })
 
     .controller("PKIController", function($scope, $rootScope, Certificate, CACertificate, Team, $location, $http) {
diff --git a/admin/static/views/settings.html b/admin/static/views/settings.html
index c7ea853c..d00d1459 100644
--- a/admin/static/views/settings.html
+++ b/admin/static/views/settings.html
@@ -204,7 +204,16 @@
 	Dernier import&nbsp;: {{ syncReport._date[1] }}
       </div>
       <div class="text-left" ng-if="configro.sync">
-	<button type="button" class="btn btn-secondary" ng-click="deepSync()" ng-disabled="deepSyncInProgress"><span class="glyphicon glyphicon-import" aria-hidden="true"></span> Synchronisation intégrale</button>
+	<div class="btn-group">
+	  <button type="button" class="btn btn-secondary" ng-click="deepSync()" ng-disabled="deepSyncInProgress"><span class="glyphicon glyphicon-import" aria-hidden="true"></span> Synchronisation intégrale</button>
+	  <button type="button" class="btn btn-secondary dropdown-toggle dropdown-toggle-split" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" ng-disabled="deepSyncInProgress">
+	    <span class="sr-only">Toggle Dropdown</span>
+	  </button>
+	  <div class="dropdown-menu" ng-controller="ThemesListController">
+	    <a class="dropdown-item" ng-click="deepSync(theme)" ng-repeat="theme in themes" ng-bind="theme.name"></a>
+	  </div>
+	</div>
+	<button type="button" class="btn btn-secondary" ng-click="speedyDeepSync()" ng-disabled="deepSyncInProgress"><span class="glyphicon glyphicon-import" aria-hidden="true"></span> Synchronisation sans fichiers</button>
 	<a href="check_import.html" class="btn btn-success" target="_self" ng-if="syncReport">Voir le rapport</a>
       </div>
     </div>
diff --git a/admin/sync/full.go b/admin/sync/full.go
index a361abdd..1fa71113 100644
--- a/admin/sync/full.go
+++ b/admin/sync/full.go
@@ -17,10 +17,60 @@ var DeepReportPath = "full_import_report.json"
 
 // oneDeepSync ensure there is no more than one running deep sync.
 var oneDeepSync sync.Mutex
+var oneThemeDeepSync sync.Mutex
 
 // DeepSyncProgress expose the progression of the depp synchronization (0 = 0%, 255 = 100%).
 var DeepSyncProgress uint8
 
+// SpeedySyncDeep performs a recursive synchronisation without importing files.
+func SpeedySyncDeep(i Importer) (errs map[string][]string) {
+	oneDeepSync.Lock()
+	defer func(){
+		oneDeepSync.Unlock()
+		if DeepSyncProgress != 255 {
+			log.Printf("Speedy synchronization terminated at step %d/255", DeepSyncProgress)
+		}
+	}()
+	DeepSyncProgress = 1
+
+	errs = map[string][]string{}
+
+	startTime := time.Now()
+	errs["_date"] = []string{fmt.Sprintf("%v", startTime)}
+	errs["_themes"] = SyncThemes(i)
+
+	if themes, err := fic.GetThemes(); err == nil {
+		DeepSyncProgress = 2
+		var themeStep uint8 = uint8(250) / uint8(len(themes))
+
+		for tid, theme := range themes {
+			DeepSyncProgress = 3 + uint8(tid) * themeStep
+			errs[theme.Name] = SyncExercices(i, theme)
+
+			if exercices, err := theme.GetExercices(); err == nil {
+				var exerciceStep uint8 = themeStep / uint8(len(exercices))
+				for eid, exercice := range exercices {
+					log.Printf("Deep synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
+
+					DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
+					errs[theme.Name] = append(errs[theme.Name], SyncExerciceFiles(i, exercice)...)
+
+					DeepSyncProgress += exerciceStep / 2
+					_, ferrs := SyncExerciceFlags(i, exercice)
+					errs[theme.Name] = append(errs[theme.Name], ferrs...)
+				}
+			}
+		}
+	}
+
+	DeepSyncProgress = 254
+	errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
+
+	DeepSyncProgress = 255
+	log.Println("Speedy synchronization done in", time.Since(startTime))
+	return
+}
+
 // SyncDeep performs a recursive synchronisation: from themes to challenge items.
 func SyncDeep(i Importer) (errs map[string][]string) {
 	oneDeepSync.Lock()
@@ -43,34 +93,59 @@ func SyncDeep(i Importer) (errs map[string][]string) {
 		var themeStep uint8 = uint8(250) / uint8(len(themes))
 
 		for tid, theme := range themes {
-			DeepSyncProgress = 3 + uint8(tid) * themeStep
-			errs[theme.Name] = SyncExercices(i, theme)
-
-			if exercices, err := theme.GetExercices(); err == nil {
-				var exerciceStep uint8 = themeStep / uint8(len(exercices))
-				for eid, exercice := range exercices {
-					log.Printf("Full synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
-
-					DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
-					errs[theme.Name] = append(errs[theme.Name], SyncExerciceFiles(i, exercice)...)
-
-					DeepSyncProgress += exerciceStep / 3
-					flagsBindings, ferrs := SyncExerciceFlags(i, exercice)
-					errs[theme.Name] = append(errs[theme.Name], ferrs...)
-
-					DeepSyncProgress += exerciceStep / 3
-					_, herrs := SyncExerciceHints(i, exercice, flagsBindings)
-					errs[theme.Name] = append(errs[theme.Name], herrs...)
-				}
-			}
+			errs[theme.Name] = SyncThemeDeep(i, theme, tid, themeStep)
 		}
 	}
 
 	DeepSyncProgress = 254
-	errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
 
+	EditDeepReport(errs, true)
+
+	if err := settings.ForceRegeneration(); err != nil {
+		errs["_regeneration"] = append(errs["_regeneration"], err.Error())
+	}
+
+	DeepSyncProgress = 255
+	log.Println("Full synchronization done in", time.Since(startTime))
+	return
+}
+
+func readDeepReport() (ret map[string][]string, err error) {
+	if fdfrom, err := os.Open(DeepReportPath); err == nil {
+		defer fdfrom.Close()
+
+		jdec := json.NewDecoder(fdfrom)
+
+		if err := jdec.Decode(&ret); err != nil {
+			return nil, err
+		}
+	} else {
+		return nil, err
+	}
+
+	return
+}
+
+func EditDeepReport(errs map[string][]string, erase bool) {
 	errs["_regeneration"] = []string{}
 
+	if !erase {
+		if in, err := readDeepReport(); err != nil {
+			errs["_regeneration"] = append(errs["_regeneration"], err.Error())
+			log.Println(err)
+		} else {
+			for k, v := range errs {
+				in[k] = v
+			}
+
+			errs = in
+		}
+	}
+
+	if _, ok := errs["_date"]; ok {
+		errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
+	}
+
 	if fdto, err := os.Create(DeepReportPath); err == nil {
 		defer fdto.Close()
 
@@ -85,11 +160,33 @@ func SyncDeep(i Importer) (errs map[string][]string) {
 		log.Println(err)
 	}
 
-	if err := settings.ForceRegeneration(); err != nil {
-		errs["_regeneration"] = append(errs["_regeneration"], err.Error())
+}
+
+// SyncThemeDeep performs a recursive synchronisation: from challenges to challenge items.
+func SyncThemeDeep(i Importer, theme fic.Theme, tid int, themeStep uint8) (errs []string) {
+	oneThemeDeepSync.Lock()
+	defer oneThemeDeepSync.Unlock()
+
+	DeepSyncProgress = 3 + uint8(tid) * themeStep
+	errs = SyncExercices(i, theme)
+
+	if exercices, err := theme.GetExercices(); err == nil && len(exercices) > 0 {
+		var exerciceStep uint8 = themeStep / uint8(len(exercices))
+		for eid, exercice := range exercices {
+			log.Printf("Deep synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
+
+			DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
+			errs = append(errs, SyncExerciceFiles(i, exercice)...)
+
+			DeepSyncProgress += exerciceStep / 3
+			flagsBindings, ferrs := SyncExerciceFlags(i, exercice)
+			errs = append(errs, ferrs...)
+
+			DeepSyncProgress += exerciceStep / 3
+			_, herrs := SyncExerciceHints(i, exercice, flagsBindings)
+			errs = append(errs, herrs...)
+		}
 	}
 
-	DeepSyncProgress = 255
-	log.Println("Full synchronization done in", time.Since(startTime))
 	return
 }