fic/server
2
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

admin: introducing speedy deep sync and themed deep sync

Browse Source
This commit is contained in:
nemunaire 2020-01-20 08:41:36 +01:00
parent f7762c0828
commit a4c87b92a5
5 changed files with 170 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
admin/api/theme.go
View File

@ -52,10 +52,23 @@ func init() {
return map[string]interface{}{"progress": sync.DeepSyncProgress}, nil
}
}))
router.POST("/api/sync/speed", apiHandler(
func(_ httprouter.Params, _ []byte) (interface{}, error) {
st := sync.SpeedySyncDeep(sync.GlobalImporter)
sync.EditDeepReport(st, false)
return st, nil
}))
router.POST("/api/sync/deep", apiHandler(
func(_ httprouter.Params, _ []byte) (interface{}, error) {
return sync.SyncDeep(sync.GlobalImporter), nil
}))
router.POST("/api/sync/deep/:thid", apiHandler(themeHandler(
func(theme fic.Theme, _ []byte) (interface{}, error) {
st := sync.SyncThemeDeep(sync.GlobalImporter, theme, 0, 250)
sync.EditDeepReport(map[string][]string{theme.Name: st}, false)
sync.DeepSyncProgress = 255
return st, nil
})))
router.POST("/api/sync/themes", apiHandler(
func(_ httprouter.Params, _ []byte) (interface{}, error) {
return sync.SyncThemes(sync.GlobalImporter), nil

4
admin/static/check_import.html
View File

@ -8,8 +8,8 @@
</style>
<script type="text/javascript">
function disp(data) {
if (data["_date"] && data["_date"].length == 2) {
document.getElementById("date_imp").innerHTML = data["_date"][1];
if (data["_date"]) {
document.getElementById("date_imp").innerHTML = data["_date"][data["_date"].length - 1];
}
Object.keys(data).map(function(theme) {
if (data[theme] != null && theme != "_date") {

26
admin/static/js/app.js
View File

@ -554,11 +554,18 @@ angular.module("FICApp")
});
};
$scope.deepSyncInProgress = false;
$scope.deepSync = function(type) {
$rootScope.newYesNoBox('warning', 'Faire une synchronisation intégrale ?', '',
$scope.deepSync = function(theme) {
if (theme) {
question = 'Faire une synchronisation intégrale du thème ' + theme.name + ' ?'
url = "/api/sync/deep/" + theme.id
} else {
question = 'Faire une synchronisation intégrale ?'
url = "/api/sync/deep"
}
$rootScope.newYesNoBox('warning', question, '',
function() {
$scope.deepSyncInProgress = true;
$http.post("/api/sync/deep").then(function() {
$http.post(url).then(function() {
$scope.deepSyncInProgress = false;
$rootScope.newBox('success', 'Synchronisation intégrale terminée.', '<a href="check_import.html">Voir le rapport</a>.', 15000);
}, function(response) {
@ -567,6 +574,19 @@ angular.module("FICApp")
});
});
};
$scope.speedyDeepSync = function() {
$rootScope.newYesNoBox('warning', 'Faire une synchronisation profonde rapide, sans s\'occuper des fichiers ?', '',
function() {
$scope.deepSyncInProgress = true;
$http.post("/api/sync/speed").then(function() {
$scope.deepSyncInProgress = false;
$rootScope.newBox('success', 'Synchronisation profonde rapide terminée.', '<a href="check_import.html">Voir le rapport</a>.', 15000);
}, function(response) {
$scope.deepSyncInProgress = false;
$rootScope.newBox('warning', 'Synchronisation profinde rapide terminée.', '<a href="check_import.html">Voir le rapport</a>.', 15000);
});
});
};
})
.controller("PKIController", function($scope, $rootScope, Certificate, CACertificate, Team, $location, $http) {

11
admin/static/views/settings.html
View File

@ -204,7 +204,16 @@
Dernier import&nbsp;: {{ syncReport._date[1] }}
</div>
<div class="text-left" ng-if="configro.sync">
<button type="button" class="btn btn-secondary" ng-click="deepSync()" ng-disabled="deepSyncInProgress"><span class="glyphicon glyphicon-import" aria-hidden="true"></span> Synchronisation intégrale</button>
<div class="btn-group">
<button type="button" class="btn btn-secondary" ng-click="deepSync()" ng-disabled="deepSyncInProgress"><span class="glyphicon glyphicon-import" aria-hidden="true"></span> Synchronisation intégrale</button>
<button type="button" class="btn btn-secondary dropdown-toggle dropdown-toggle-split" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" ng-disabled="deepSyncInProgress">
<span class="sr-only">Toggle Dropdown</span>
</button>
<div class="dropdown-menu" ng-controller="ThemesListController">
<a class="dropdown-item" ng-click="deepSync(theme)" ng-repeat="theme in themes" ng-bind="theme.name"></a>
</div>
</div>
<button type="button" class="btn btn-secondary" ng-click="speedyDeepSync()" ng-disabled="deepSyncInProgress"><span class="glyphicon glyphicon-import" aria-hidden="true"></span> Synchronisation sans fichiers</button>
<a href="check_import.html" class="btn btn-success" target="_self" ng-if="syncReport">Voir le rapport</a>
</div>
</div>

147
admin/sync/full.go
View File

@ -17,10 +17,60 @@ var DeepReportPath = "full_import_report.json"
// oneDeepSync ensure there is no more than one running deep sync.
var oneDeepSync sync.Mutex
var oneThemeDeepSync sync.Mutex
// DeepSyncProgress expose the progression of the depp synchronization (0 = 0%, 255 = 100%).
var DeepSyncProgress uint8
// SpeedySyncDeep performs a recursive synchronisation without importing files.
func SpeedySyncDeep(i Importer) (errs map[string][]string) {
oneDeepSync.Lock()
defer func(){
oneDeepSync.Unlock()
if DeepSyncProgress != 255 {
log.Printf("Speedy synchronization terminated at step %d/255", DeepSyncProgress)
}
}()
DeepSyncProgress = 1
errs = map[string][]string{}
startTime := time.Now()
errs["_date"] = []string{fmt.Sprintf("%v", startTime)}
errs["_themes"] = SyncThemes(i)
if themes, err := fic.GetThemes(); err == nil {
DeepSyncProgress = 2
var themeStep uint8 = uint8(250) / uint8(len(themes))
for tid, theme := range themes {
DeepSyncProgress = 3 + uint8(tid) * themeStep
errs[theme.Name] = SyncExercices(i, theme)
if exercices, err := theme.GetExercices(); err == nil {
var exerciceStep uint8 = themeStep / uint8(len(exercices))
for eid, exercice := range exercices {
log.Printf("Deep synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
errs[theme.Name] = append(errs[theme.Name], SyncExerciceFiles(i, exercice)...)
DeepSyncProgress += exerciceStep / 2
_, ferrs := SyncExerciceFlags(i, exercice)
errs[theme.Name] = append(errs[theme.Name], ferrs...)
}
}
}
}
DeepSyncProgress = 254
errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
DeepSyncProgress = 255
log.Println("Speedy synchronization done in", time.Since(startTime))
return
}
// SyncDeep performs a recursive synchronisation: from themes to challenge items.
func SyncDeep(i Importer) (errs map[string][]string) {
oneDeepSync.Lock()
@ -43,34 +93,59 @@ func SyncDeep(i Importer) (errs map[string][]string) {
var themeStep uint8 = uint8(250) / uint8(len(themes))
for tid, theme := range themes {
DeepSyncProgress = 3 + uint8(tid) * themeStep
errs[theme.Name] = SyncExercices(i, theme)
if exercices, err := theme.GetExercices(); err == nil {
var exerciceStep uint8 = themeStep / uint8(len(exercices))
for eid, exercice := range exercices {
log.Printf("Full synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
errs[theme.Name] = append(errs[theme.Name], SyncExerciceFiles(i, exercice)...)
DeepSyncProgress += exerciceStep / 3
flagsBindings, ferrs := SyncExerciceFlags(i, exercice)
errs[theme.Name] = append(errs[theme.Name], ferrs...)
DeepSyncProgress += exerciceStep / 3
_, herrs := SyncExerciceHints(i, exercice, flagsBindings)
errs[theme.Name] = append(errs[theme.Name], herrs...)
}
}
errs[theme.Name] = SyncThemeDeep(i, theme, tid, themeStep)
}
}
DeepSyncProgress = 254
errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
EditDeepReport(errs, true)
if err := settings.ForceRegeneration(); err != nil {
errs["_regeneration"] = append(errs["_regeneration"], err.Error())
}
DeepSyncProgress = 255
log.Println("Full synchronization done in", time.Since(startTime))
return
}
func readDeepReport() (ret map[string][]string, err error) {
if fdfrom, err := os.Open(DeepReportPath); err == nil {
defer fdfrom.Close()
jdec := json.NewDecoder(fdfrom)
if err := jdec.Decode(&ret); err != nil {
return nil, err
}
} else {
return nil, err
}
return
}
func EditDeepReport(errs map[string][]string, erase bool) {
errs["_regeneration"] = []string{}
if !erase {
if in, err := readDeepReport(); err != nil {
errs["_regeneration"] = append(errs["_regeneration"], err.Error())
log.Println(err)
} else {
for k, v := range errs {
in[k] = v
}
errs = in
}
}
if _, ok := errs["_date"]; ok {
errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
}
if fdto, err := os.Create(DeepReportPath); err == nil {
defer fdto.Close()
@ -85,11 +160,33 @@ func SyncDeep(i Importer) (errs map[string][]string) {
log.Println(err)
}
if err := settings.ForceRegeneration(); err != nil {
errs["_regeneration"] = append(errs["_regeneration"], err.Error())
}
// SyncThemeDeep performs a recursive synchronisation: from challenges to challenge items.
func SyncThemeDeep(i Importer, theme fic.Theme, tid int, themeStep uint8) (errs []string) {
oneThemeDeepSync.Lock()
defer oneThemeDeepSync.Unlock()
DeepSyncProgress = 3 + uint8(tid) * themeStep
errs = SyncExercices(i, theme)
if exercices, err := theme.GetExercices(); err == nil && len(exercices) > 0 {
var exerciceStep uint8 = themeStep / uint8(len(exercices))
for eid, exercice := range exercices {
log.Printf("Deep synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
errs = append(errs, SyncExerciceFiles(i, exercice)...)
DeepSyncProgress += exerciceStep / 3
flagsBindings, ferrs := SyncExerciceFlags(i, exercice)
errs = append(errs, ferrs...)
DeepSyncProgress += exerciceStep / 3
_, herrs := SyncExerciceHints(i, exercice, flagsBindings)
errs = append(errs, herrs...)
}
}
DeepSyncProgress = 255
log.Println("Full synchronization done in", time.Since(startTime))
return
}