admin: introducing speedy deep sync and themed deep sync
This commit is contained in:
parent
f7762c0828
commit
a4c87b92a5
@ -52,10 +52,23 @@ func init() {
|
|||||||
return map[string]interface{}{"progress": sync.DeepSyncProgress}, nil
|
return map[string]interface{}{"progress": sync.DeepSyncProgress}, nil
|
||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
|
router.POST("/api/sync/speed", apiHandler(
|
||||||
|
func(_ httprouter.Params, _ []byte) (interface{}, error) {
|
||||||
|
st := sync.SpeedySyncDeep(sync.GlobalImporter)
|
||||||
|
sync.EditDeepReport(st, false)
|
||||||
|
return st, nil
|
||||||
|
}))
|
||||||
router.POST("/api/sync/deep", apiHandler(
|
router.POST("/api/sync/deep", apiHandler(
|
||||||
func(_ httprouter.Params, _ []byte) (interface{}, error) {
|
func(_ httprouter.Params, _ []byte) (interface{}, error) {
|
||||||
return sync.SyncDeep(sync.GlobalImporter), nil
|
return sync.SyncDeep(sync.GlobalImporter), nil
|
||||||
}))
|
}))
|
||||||
|
router.POST("/api/sync/deep/:thid", apiHandler(themeHandler(
|
||||||
|
func(theme fic.Theme, _ []byte) (interface{}, error) {
|
||||||
|
st := sync.SyncThemeDeep(sync.GlobalImporter, theme, 0, 250)
|
||||||
|
sync.EditDeepReport(map[string][]string{theme.Name: st}, false)
|
||||||
|
sync.DeepSyncProgress = 255
|
||||||
|
return st, nil
|
||||||
|
})))
|
||||||
router.POST("/api/sync/themes", apiHandler(
|
router.POST("/api/sync/themes", apiHandler(
|
||||||
func(_ httprouter.Params, _ []byte) (interface{}, error) {
|
func(_ httprouter.Params, _ []byte) (interface{}, error) {
|
||||||
return sync.SyncThemes(sync.GlobalImporter), nil
|
return sync.SyncThemes(sync.GlobalImporter), nil
|
||||||
|
@ -8,8 +8,8 @@
|
|||||||
</style>
|
</style>
|
||||||
<script type="text/javascript">
|
<script type="text/javascript">
|
||||||
function disp(data) {
|
function disp(data) {
|
||||||
if (data["_date"] && data["_date"].length == 2) {
|
if (data["_date"]) {
|
||||||
document.getElementById("date_imp").innerHTML = data["_date"][1];
|
document.getElementById("date_imp").innerHTML = data["_date"][data["_date"].length - 1];
|
||||||
}
|
}
|
||||||
Object.keys(data).map(function(theme) {
|
Object.keys(data).map(function(theme) {
|
||||||
if (data[theme] != null && theme != "_date") {
|
if (data[theme] != null && theme != "_date") {
|
||||||
|
@ -554,11 +554,18 @@ angular.module("FICApp")
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
$scope.deepSyncInProgress = false;
|
$scope.deepSyncInProgress = false;
|
||||||
$scope.deepSync = function(type) {
|
$scope.deepSync = function(theme) {
|
||||||
$rootScope.newYesNoBox('warning', 'Faire une synchronisation intégrale ?', '',
|
if (theme) {
|
||||||
|
question = 'Faire une synchronisation intégrale du thème ' + theme.name + ' ?'
|
||||||
|
url = "/api/sync/deep/" + theme.id
|
||||||
|
} else {
|
||||||
|
question = 'Faire une synchronisation intégrale ?'
|
||||||
|
url = "/api/sync/deep"
|
||||||
|
}
|
||||||
|
$rootScope.newYesNoBox('warning', question, '',
|
||||||
function() {
|
function() {
|
||||||
$scope.deepSyncInProgress = true;
|
$scope.deepSyncInProgress = true;
|
||||||
$http.post("/api/sync/deep").then(function() {
|
$http.post(url).then(function() {
|
||||||
$scope.deepSyncInProgress = false;
|
$scope.deepSyncInProgress = false;
|
||||||
$rootScope.newBox('success', 'Synchronisation intégrale terminée.', '<a href="check_import.html">Voir le rapport</a>.', 15000);
|
$rootScope.newBox('success', 'Synchronisation intégrale terminée.', '<a href="check_import.html">Voir le rapport</a>.', 15000);
|
||||||
}, function(response) {
|
}, function(response) {
|
||||||
@ -567,6 +574,19 @@ angular.module("FICApp")
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
$scope.speedyDeepSync = function() {
|
||||||
|
$rootScope.newYesNoBox('warning', 'Faire une synchronisation profonde rapide, sans s\'occuper des fichiers ?', '',
|
||||||
|
function() {
|
||||||
|
$scope.deepSyncInProgress = true;
|
||||||
|
$http.post("/api/sync/speed").then(function() {
|
||||||
|
$scope.deepSyncInProgress = false;
|
||||||
|
$rootScope.newBox('success', 'Synchronisation profonde rapide terminée.', '<a href="check_import.html">Voir le rapport</a>.', 15000);
|
||||||
|
}, function(response) {
|
||||||
|
$scope.deepSyncInProgress = false;
|
||||||
|
$rootScope.newBox('warning', 'Synchronisation profinde rapide terminée.', '<a href="check_import.html">Voir le rapport</a>.', 15000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
})
|
})
|
||||||
|
|
||||||
.controller("PKIController", function($scope, $rootScope, Certificate, CACertificate, Team, $location, $http) {
|
.controller("PKIController", function($scope, $rootScope, Certificate, CACertificate, Team, $location, $http) {
|
||||||
|
@ -204,7 +204,16 @@
|
|||||||
Dernier import : {{ syncReport._date[1] }}
|
Dernier import : {{ syncReport._date[1] }}
|
||||||
</div>
|
</div>
|
||||||
<div class="text-left" ng-if="configro.sync">
|
<div class="text-left" ng-if="configro.sync">
|
||||||
<button type="button" class="btn btn-secondary" ng-click="deepSync()" ng-disabled="deepSyncInProgress"><span class="glyphicon glyphicon-import" aria-hidden="true"></span> Synchronisation intégrale</button>
|
<div class="btn-group">
|
||||||
|
<button type="button" class="btn btn-secondary" ng-click="deepSync()" ng-disabled="deepSyncInProgress"><span class="glyphicon glyphicon-import" aria-hidden="true"></span> Synchronisation intégrale</button>
|
||||||
|
<button type="button" class="btn btn-secondary dropdown-toggle dropdown-toggle-split" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" ng-disabled="deepSyncInProgress">
|
||||||
|
<span class="sr-only">Toggle Dropdown</span>
|
||||||
|
</button>
|
||||||
|
<div class="dropdown-menu" ng-controller="ThemesListController">
|
||||||
|
<a class="dropdown-item" ng-click="deepSync(theme)" ng-repeat="theme in themes" ng-bind="theme.name"></a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<button type="button" class="btn btn-secondary" ng-click="speedyDeepSync()" ng-disabled="deepSyncInProgress"><span class="glyphicon glyphicon-import" aria-hidden="true"></span> Synchronisation sans fichiers</button>
|
||||||
<a href="check_import.html" class="btn btn-success" target="_self" ng-if="syncReport">Voir le rapport</a>
|
<a href="check_import.html" class="btn btn-success" target="_self" ng-if="syncReport">Voir le rapport</a>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
@ -17,10 +17,60 @@ var DeepReportPath = "full_import_report.json"
|
|||||||
|
|
||||||
// oneDeepSync ensure there is no more than one running deep sync.
|
// oneDeepSync ensure there is no more than one running deep sync.
|
||||||
var oneDeepSync sync.Mutex
|
var oneDeepSync sync.Mutex
|
||||||
|
var oneThemeDeepSync sync.Mutex
|
||||||
|
|
||||||
// DeepSyncProgress expose the progression of the depp synchronization (0 = 0%, 255 = 100%).
|
// DeepSyncProgress expose the progression of the depp synchronization (0 = 0%, 255 = 100%).
|
||||||
var DeepSyncProgress uint8
|
var DeepSyncProgress uint8
|
||||||
|
|
||||||
|
// SpeedySyncDeep performs a recursive synchronisation without importing files.
|
||||||
|
func SpeedySyncDeep(i Importer) (errs map[string][]string) {
|
||||||
|
oneDeepSync.Lock()
|
||||||
|
defer func(){
|
||||||
|
oneDeepSync.Unlock()
|
||||||
|
if DeepSyncProgress != 255 {
|
||||||
|
log.Printf("Speedy synchronization terminated at step %d/255", DeepSyncProgress)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
DeepSyncProgress = 1
|
||||||
|
|
||||||
|
errs = map[string][]string{}
|
||||||
|
|
||||||
|
startTime := time.Now()
|
||||||
|
errs["_date"] = []string{fmt.Sprintf("%v", startTime)}
|
||||||
|
errs["_themes"] = SyncThemes(i)
|
||||||
|
|
||||||
|
if themes, err := fic.GetThemes(); err == nil {
|
||||||
|
DeepSyncProgress = 2
|
||||||
|
var themeStep uint8 = uint8(250) / uint8(len(themes))
|
||||||
|
|
||||||
|
for tid, theme := range themes {
|
||||||
|
DeepSyncProgress = 3 + uint8(tid) * themeStep
|
||||||
|
errs[theme.Name] = SyncExercices(i, theme)
|
||||||
|
|
||||||
|
if exercices, err := theme.GetExercices(); err == nil {
|
||||||
|
var exerciceStep uint8 = themeStep / uint8(len(exercices))
|
||||||
|
for eid, exercice := range exercices {
|
||||||
|
log.Printf("Deep synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
|
||||||
|
|
||||||
|
DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
|
||||||
|
errs[theme.Name] = append(errs[theme.Name], SyncExerciceFiles(i, exercice)...)
|
||||||
|
|
||||||
|
DeepSyncProgress += exerciceStep / 2
|
||||||
|
_, ferrs := SyncExerciceFlags(i, exercice)
|
||||||
|
errs[theme.Name] = append(errs[theme.Name], ferrs...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DeepSyncProgress = 254
|
||||||
|
errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
|
||||||
|
|
||||||
|
DeepSyncProgress = 255
|
||||||
|
log.Println("Speedy synchronization done in", time.Since(startTime))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// SyncDeep performs a recursive synchronisation: from themes to challenge items.
|
// SyncDeep performs a recursive synchronisation: from themes to challenge items.
|
||||||
func SyncDeep(i Importer) (errs map[string][]string) {
|
func SyncDeep(i Importer) (errs map[string][]string) {
|
||||||
oneDeepSync.Lock()
|
oneDeepSync.Lock()
|
||||||
@ -43,34 +93,59 @@ func SyncDeep(i Importer) (errs map[string][]string) {
|
|||||||
var themeStep uint8 = uint8(250) / uint8(len(themes))
|
var themeStep uint8 = uint8(250) / uint8(len(themes))
|
||||||
|
|
||||||
for tid, theme := range themes {
|
for tid, theme := range themes {
|
||||||
DeepSyncProgress = 3 + uint8(tid) * themeStep
|
errs[theme.Name] = SyncThemeDeep(i, theme, tid, themeStep)
|
||||||
errs[theme.Name] = SyncExercices(i, theme)
|
|
||||||
|
|
||||||
if exercices, err := theme.GetExercices(); err == nil {
|
|
||||||
var exerciceStep uint8 = themeStep / uint8(len(exercices))
|
|
||||||
for eid, exercice := range exercices {
|
|
||||||
log.Printf("Full synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
|
|
||||||
|
|
||||||
DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
|
|
||||||
errs[theme.Name] = append(errs[theme.Name], SyncExerciceFiles(i, exercice)...)
|
|
||||||
|
|
||||||
DeepSyncProgress += exerciceStep / 3
|
|
||||||
flagsBindings, ferrs := SyncExerciceFlags(i, exercice)
|
|
||||||
errs[theme.Name] = append(errs[theme.Name], ferrs...)
|
|
||||||
|
|
||||||
DeepSyncProgress += exerciceStep / 3
|
|
||||||
_, herrs := SyncExerciceHints(i, exercice, flagsBindings)
|
|
||||||
errs[theme.Name] = append(errs[theme.Name], herrs...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
DeepSyncProgress = 254
|
DeepSyncProgress = 254
|
||||||
errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
|
|
||||||
|
|
||||||
|
EditDeepReport(errs, true)
|
||||||
|
|
||||||
|
if err := settings.ForceRegeneration(); err != nil {
|
||||||
|
errs["_regeneration"] = append(errs["_regeneration"], err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
DeepSyncProgress = 255
|
||||||
|
log.Println("Full synchronization done in", time.Since(startTime))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func readDeepReport() (ret map[string][]string, err error) {
|
||||||
|
if fdfrom, err := os.Open(DeepReportPath); err == nil {
|
||||||
|
defer fdfrom.Close()
|
||||||
|
|
||||||
|
jdec := json.NewDecoder(fdfrom)
|
||||||
|
|
||||||
|
if err := jdec.Decode(&ret); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func EditDeepReport(errs map[string][]string, erase bool) {
|
||||||
errs["_regeneration"] = []string{}
|
errs["_regeneration"] = []string{}
|
||||||
|
|
||||||
|
if !erase {
|
||||||
|
if in, err := readDeepReport(); err != nil {
|
||||||
|
errs["_regeneration"] = append(errs["_regeneration"], err.Error())
|
||||||
|
log.Println(err)
|
||||||
|
} else {
|
||||||
|
for k, v := range errs {
|
||||||
|
in[k] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
errs = in
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := errs["_date"]; ok {
|
||||||
|
errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
|
||||||
|
}
|
||||||
|
|
||||||
if fdto, err := os.Create(DeepReportPath); err == nil {
|
if fdto, err := os.Create(DeepReportPath); err == nil {
|
||||||
defer fdto.Close()
|
defer fdto.Close()
|
||||||
|
|
||||||
@ -85,11 +160,33 @@ func SyncDeep(i Importer) (errs map[string][]string) {
|
|||||||
log.Println(err)
|
log.Println(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := settings.ForceRegeneration(); err != nil {
|
}
|
||||||
errs["_regeneration"] = append(errs["_regeneration"], err.Error())
|
|
||||||
|
// SyncThemeDeep performs a recursive synchronisation: from challenges to challenge items.
|
||||||
|
func SyncThemeDeep(i Importer, theme fic.Theme, tid int, themeStep uint8) (errs []string) {
|
||||||
|
oneThemeDeepSync.Lock()
|
||||||
|
defer oneThemeDeepSync.Unlock()
|
||||||
|
|
||||||
|
DeepSyncProgress = 3 + uint8(tid) * themeStep
|
||||||
|
errs = SyncExercices(i, theme)
|
||||||
|
|
||||||
|
if exercices, err := theme.GetExercices(); err == nil && len(exercices) > 0 {
|
||||||
|
var exerciceStep uint8 = themeStep / uint8(len(exercices))
|
||||||
|
for eid, exercice := range exercices {
|
||||||
|
log.Printf("Deep synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
|
||||||
|
|
||||||
|
DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
|
||||||
|
errs = append(errs, SyncExerciceFiles(i, exercice)...)
|
||||||
|
|
||||||
|
DeepSyncProgress += exerciceStep / 3
|
||||||
|
flagsBindings, ferrs := SyncExerciceFlags(i, exercice)
|
||||||
|
errs = append(errs, ferrs...)
|
||||||
|
|
||||||
|
DeepSyncProgress += exerciceStep / 3
|
||||||
|
_, herrs := SyncExerciceHints(i, exercice, flagsBindings)
|
||||||
|
errs = append(errs, herrs...)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
DeepSyncProgress = 255
|
|
||||||
log.Println("Full synchronization done in", time.Since(startTime))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user