https://sgdml.org/js/dataset_upload.js

1var max_size = 500 * 1024 * 1024; // Max upload size: 500MB
2var slice_size = 512 * 1024; // Max size of each fragment: 512 KB
3var concurrency = 1; // Max thread number 
4var file = {}; // Pointer to file to upload
5var abort = false; // Flag used to abort operation
6var goals = 0; // Counter of parallel threads completed
7var percent = 0; // Percent of progress
8var sessionID = null;
9
10var updateInterval = null;
11var md5 = null;
12var datasetID = null;
13
14var isServiceRunning = true;
15
16//const dataStoreURL = 'http://www.quantum-machine.org/gdml/';
17const dataStoreURL = 'https://sgdml.org/secure_proxy.php?file=';
18const dataStoreURLproxy = 'https://www.sgdml.org/';
19
20$(document).ready(function(e) {
21
22    checkServiceStatus();
23    setInterval(function() {
24        checkServiceStatus();
25    }, 60000);
26
27    var url = window.location.href;
28    if (url.indexOf('?datasetID=') != -1) {
29
30        const urlParams = new URLSearchParams(window.location.search);
31        datasetID = urlParams.get('datasetID');
32
33        initModal();
34    }
35
36    $('#datasetUploadForm').on('submit', function(e) {
37        e.preventDefault(); // Prevent standard submit.
38
39        file = document.querySelector('#dbi-file-upload').files[0];
40        if (file.size > max_size) {
41            alert('Warning! \nFiles bigger than 500MB not supported');
42            return;
43        } else if (file.name.endsWith('.part')) {
44            alert('Warning! \nFiles with .part extension not supported');
45            return;
46        } else if (!file.name.endsWith('.npz')) {
47            alert('Warning! \nOnly files with .npz extension are supported.');
48            return;
49        }
50
51        init_upload_file();
52    });
53
54    $('#datasetUploadModal').on('show.bs.modal', function() {
55
56        // start polling for updates every 15 seconds
57        updateInterval = setInterval(function() {
58            initModal();
59        }, 15000);
60    });
61
62    $('#datasetUploadModal').on('hidden.bs.modal', function() {
63
64        // Reset job submission form.
65        inputReset('numberTrainingPoints');
66        enableTrainigJobSubmission();
67        showError();
68        $('#submitTrainingJobFailed').css('display', 'none');
69
70        // Collapse dataset description accordion.
71        $('#properties .collapse').collapse('hide');
72
73        removeParametersFromURL();
74        md5 = null;
75
76        clearInterval(updateInterval);
77    });
78});
79
80
81function showServiceStatus() {
82
83    $('#service_badge_default').css('display', isServiceRunning ? 'block' : 'none');
84    $('#service_badge_down').css('display', isServiceRunning ? 'none' : 'block');
85    $('#service_badge_down_modal').css('display', isServiceRunning ? 'none' : 'block');
86
87    if (isServiceRunning) {
88
89        //$('#dbi-upload-submit').removeAttr('disabled');
90        //$('#dbi-file-upload').removeAttr('disabled');
91
92        enableTrainigJobSubmission();
93    } else {
94
95        //$('#dbi-upload-submit').attr('disabled', 'disabled');
96        //$('#dbi-file-upload').attr('disabled', 'disabled');
97
98        enableTrainigJobSubmission(false);
99    }
100}
101
102function checkServiceStatus() {
103
104    $.getJSON(dataStoreURL + 'services/get_status.php', function(status) {
105        isServiceRunning = status['is_service_running'];
106        showServiceStatus();
107    }).fail(function(d, textStatus, error) {
108        isServiceRunning = false;
109        showServiceStatus();
110    });
111}
112
113function addDatasetIDToURL(datasetID) {
114
115    var url = window.location.href.split(/[?#]/)[0];
116    window.history.pushState(null, 'null', url + '?datasetID=' + datasetID)
117}
118
119function removeParametersFromURL() {
120
121    var url = window.location.href.split(/[?#]/)[0];
122    window.history.pushState(null, "null", url)
123}
124
125function copyDatasetUploadModalURLToClipboard() {
126
127    var copyText = document.getElementById('datasetUploadModalURL');
128    copyText.select();
129    copyText.setSelectionRange(0, 99999); // for mobile devices
130
131    document.execCommand('copy');
132}
133
134function orderedPosition(table, newNTrain) {
135
136    // Return ordered position of new row.
137    for (var i = 0, row; row = table.rows[i]; i++) {
138        nTrain = parseInt(row.cells[0].innerHTML.replace(',', ''));
139
140        if (!isNaN(nTrain)) {
141            if (nTrain < newNTrain) {
142                break;
143            }
144        }
145    }
146
147    return i;
148}
149
150function deleteRow(table, nTrain) {
151
152    // Return ordered position of new row.
153    for (var i = 0, row; row = table.rows[i]; i++) {

154        if (nTrain == parseInt(row.cells[0].innerHTML.replace(',', ''))) {
155            table.deleteRow(i);
156            break;
157        }
158    }
159}
160
161function editModelRow(tableID, model, row=null) {
162
163    // Add new row if no existing row was provided.
164    if (row == null) {
165
166        const table = document.getElementById(tableID);
167        orderedPos = orderedPosition(table, model['n_train'])
168        row = table.insertRow(orderedPos);
169
170        for (col = 0; col < 4; col++) {
171            row.insertCell();
172        }
173    }
174
175    col = row.cells[0];
176    populateTrainingPointsCell(col, model['n_train'])
177
178    isConv = ('is_conv' in model && model['is_conv']) || !('is_conv' in model);
179
180    col = row.cells[1];
181    col.innerHTML = '';
182    if ('e_err' in model) {
183        e_mae = (model['e_err'] == null || !isConv) ? null : model['e_err']['mae'];
184        populateTestErrorCell(col, e_mae);
185    }
186
187    f_mae = null;
188    col = row.cells[2];
189    col.innerHTML = '';
190    if ('f_err' in model) {
191        f_mae = (model['f_err'] == null || !isConv) ? null : model['f_err']['mae'];
192        populateTestErrorCell(col, f_mae);
193    }
194
195    col = row.cells[3];
196    isJob = !('f_err' in model) || ('f_err' in model && model['f_err'] == null)
197    populateStatusCell(col, model['status'], isJob, isConv ? model['model_url'] : null, model['log_url']);
198}
199
200function populateTrainingPointsCell(cell, nTrain) {
201
202    col.setAttribute('align', 'right');
203
204    // spaces as thousands separators
205    cell.innerHTML = String(nTrain).replace(/\B(?=(\d{3})+(?!\d))/g, ',');
206}
207
208function populateTestErrorCell(cell, error) {
209
210    cell.setAttribute('align', 'right');
211
212    //const err_str = (error == null) ? 'n/a' : parseFloat(error).toFixed(4);
213    //const err_str_short = (error == null) ? 'n/a' : parseFloat(error).toFixed(2);
214
215    const err_str = (error == null) ? '<span class="badge badge-warning">NOT CONV!</span>' : parseFloat(error).toFixed(4);
216    const err_str_short = (error == null) ? '<span class="badge badge-warning">NC!</span>' : parseFloat(error).toFixed(2);
217
218    cell.innerHTML = '<span class="d-lg-none">' + err_str_short + '</span> <span class="d-none d-lg-block">' + err_str + '</span></a>'
219}
220
221function populateStatusCell(cell, status, is_job, modelURL = null, logURL = null) {
222
223    cell.setAttribute('align', 'right');
224
225    if (is_job) {
226        if (status == 'R') {
227            cell.innerHTML = '<span style="font-size:14px" class="text-secondary"> Training</span> <div style="width: 10px; height:10px; position:relative; bottom:2px;" class="spinner-grow text-success spinner-grow-sm" role="status"></div>';
228        } else if (status == 'PD') {
229            cell.innerHTML = '<span style="font-size:14px" class="text-secondary"> Queued</span> <span class="status_dot_yellow"></span>';
230        } else if (status == 'REF') {
231            cell.innerHTML = '<span style="font-size:14px" class="text-secondary"> Scheduled</span> <span class="status_dot_green"></span>';
232        } else if (status == 'ERR') {
233            //cell.innerHTML = '<span style="font-size:14px" class="text-secondary"> Failed</span> <span class="status_dot_red"></span>';
234            cell.innerHTML = '<a class="btn btn-outline-danger btn-sm ' + (logURL == null ? 'disabled' : '') + '"  target="_blank" role="button" href="' + dataStoreURL + logURL + '"><span class="d-none d-lg-inline">Error </span>Log</a>';
235        } else {
236            cell.innerHTML = '<span style="font-size:14px" class="text-secondary"> Waiting<span class="d-none d-lg-inline"> (to start/resume)</span></span> <span class="status_dot_gray"></span>';
237        }
238
239    } else {
240
241        cell.innerHTML = '';
242
243        var div = document.createElement('div');
244        div.setAttribute('class', 'btn-group');
245        div.setAttribute('role', 'group');
246
247        div.innerHTML = '<a class="btn btn-outline-primary btn-sm ' + (modelURL == null ? 'disabled' : '') + '" role="button" href="' + dataStoreURL + modelURL + '">Model</a>';
248        div.innerHTML += '<a class="btn btn-outline-secondary btn-sm ' + (logURL == null ? 'disabled' : '') + '"  target="_blank" role="button" href="' + dataStoreURL + logURL + '"><span class="d-none d-lg-inline">Training </span> Log</a>';
249
250        cell.appendChild(div);
251    }
252}
253
254function loadMolViewer(xyz_str) {
255
256    let element = $('#container-01');

257    let config = {
258        backgroundColor: 'white',
259        minimumZoomToDistance: 1
260    };
261    let viewer = $3Dmol.createViewer(element, config);
262    
263    viewer.addModel(xyz_str, 'xyz');
264
265    viewer.addStyle({}, {
266        stick: {
267            radius: 0.15
268        },
269        sphere: {
270            scale: 0.25,
271            colorscheme: "Jmol"
272        }
273    });
274    viewer.addUnitCell();
275
276    viewer.zoomTo();
277    viewer.render();
278
279    viewer.spin('vy', 0.25);
280}
281
282
283function openModal(_datasetID) {
284
285    datasetID = _datasetID;
286    md5 = null;
287
288    // show correct URL in address bar
289    addDatasetIDToURL(_datasetID);
290
291    initModal();
292}
293
294
295function initModal() {
296
297    if (md5 == null && datasetID != null) { // still waiting for dataset file to be processed
298
299        document.getElementById('datasetUploadModalURL').value = 'http://sgdml.org?datasetID=' + datasetID;
300        document.getElementById('jobSubmissionDatasetID').value = datasetID;
301
302        $.getJSON(dataStoreURLproxy + 'secure_proxy.php?service=get_dataset_metadata&datasetID=' + datasetID, function(dataset) {
303
304            if ('error' in dataset) {
305                showDatasetProcessingFailView();
306            } else if (!('md5' in dataset)) {
307                datasetProcessingView();
308            } else {
309                fullView(dataset, datasetID)
310
311                md5 = dataset['md5'];
312                updateModelTable(md5);
313            }
314            
315            const modalID = '#datasetUploadModal'
316            var isShown = $(modalID).hasClass('in') || $(modalID).hasClass('show')
317            if (!isShown) {
318                $(modalID).modal('show');
319            }
320
321        }).fail(function(d, textStatus, error) {
322            //console.error('getJSON failed, status: ' + textStatus + ', error: ' + error);
323
324            alert('Requested dataset ID not found.');
325
326            removeParametersFromURL();
327        });
328
329    } else {
330
331        if (md5 == null) {
332            console.log('Error: MD5 not set!!')
333        }
334
335        updateModelTable(md5);
336    }
337}
338
339
340function updateModelTable(md5) {
341
342    const tableID = 'datasetUploadModalTrainedModels';
343    var table = document.getElementById(tableID);
344
345    $.getJSON(dataStoreURLproxy + 'secure_proxy.php?service=get_models&md5=' + md5, function(models) {
346
347        var deleteNTrains = []
348        for (var i = 0, row; row = table.rows[i]; i++) {
349
350            // Get index of row with this number of training poitns.
351            nTrain = parseInt(row.cells[0].innerHTML.replace(',', ''));
352            index = models.findIndex(function(model) {
353                return model['n_train'] == nTrain;
354            });
355
356            // Update row.
357            if (index >= 0) {
358
359                model = models[index];
360                editModelRow(tableID, model, row);
361
362            } else if (!isNaN(nTrain)) { // Skip rows without an integer in the first column (i.e. header and form).
363
364                deleteNTrains.push(nTrain);
365            }
366        }
367
368        // Delete unneeded rows.
369        deleteNTrains.forEach(function(nTrain){
370            deleteRow(table, nTrain);
371        });
372
373        // Numbers of training points for each model/job.
374        model_ntrains = models.map(function(model) {
375            return model['n_train'];
376        });
377
378        // Numbers of training points in table rows.
379        var table_ntrains = []
380        for (var i = 0, row; row = table.rows[i]; i++) {
381            nTrain = parseInt(row.cells[0].innerHTML.replace(',', ''));
382            if (!isNaN(nTrain) && typeof nTrain !== 'undefined')
383                table_ntrains.push(nTrain);
384        }
385
386        // Insert missing rows.
387        missingModelsInTable = models.filter(model => !table_ntrains.includes(model['n_train']));
388        missingModelsInTable.forEach(function(model) {
389            editModelRow(tableID, model);
390        });
391
392    }).fail(function(d, textStatus, error) {
393        console.error('updateOverview: getJSON model failed, status: ' + textStatus + ', error: ' + error);
394    });
395}
396
397function init_upload_file() {
398    // Global browser-side variables init, form disable and some console logging
399    abort = false;

400    goals = 0;
401    percent = 0;
402
403    $('#dbi-upload-submit').attr('disabled', 'disabled');
404    $('#dbi-upload-progress').html('Upload init...');
405
406    $('#progress').css('display', 'block');
407
408    // First call to server with 'init' parameter
409    var formData = new FormData();
410    formData.append('init', 1);
411
412    $.ajax({
413        url: dataStoreURL + 'upload.php',
414        type: 'POST',
415        dataType: 'json',
416        contentType: false,
417        cache: false,
418        processData: false,
419        data: formData,
420        error: function(data) {
421
422            $('#dbi-upload-progress').html('Init call failed'); // You can use an alert
423            $('#dbi-upload-submit').removeAttr('disabled'); // Re-enable the form submit
424
425        },
426        success: function(data) {
427
428            
429            //console.log('INIT')
430            //console.log(data)
431
432            sessionID = data['session_id']
433
434            //sessionID = null
435
436            $('#dbi-upload-progress').html('Sending file...');
437            // Calling 'core' function to send multiple fragments
438            for (var i = 0; i < concurrency; i++)
439                upload_file((i * (slice_size + 1)));
440        }
441    });
442}
443
444function upload_file(start) { // Start argument is the offset used to read a fragment
445
446    if (abort) return; // Check if some thread is in error (abort the transaction)
447
448    // Initialize file reader and put in blob var the fragment to send
449    var reader = new FileReader();
450    var next_slice = start + slice_size + 1;
451    var blob = file.slice(start, next_slice);
452
453    reader.onloadend = function(event) {
454
455        if (event.target.readyState !== FileReader.DONE) {
456            return;
457        }
458
459        // preparing the form data to send and calling the server
460        var slice_id = Math.floor(start / slice_size);
461        var formData = new FormData();
462        formData.append('fileUpload', blob, '.' + slice_id + '.part');
463        formData.append('part', slice_id);
464        formData.append('sessionID', sessionID);
465
466        $.ajax({
467            url: dataStoreURL + 'upload.php',
468            type: 'POST',
469            dataType: 'json',
470            contentType: false,
471            cache: false,
472            processData: false,
473            data: formData,
474            error: function(data) {
475
476                console.log("error: " + data);
477
478                if (abort) return; // Maybe some other thread was in error...
479
480                // If it's the first error we inform the user and re-enable the form submit
481                $('#dbi-upload-progress').html('Upload error'); // You can use an alert
482                $('#dbi-upload-submit').removeAttr('disabled');
483                abort = true;
484            },
485            success: function(data) {
486                if (abort) return; // Some other thread is in error, so we exit
487
488                // Calculate progress and next offset
489                var size_done = start + slice_size;
490                var percent_done = Math.floor((size_done / file.size) * 100);
491                var real_next_slice = start + ((slice_size + 1) * concurrency);
492                if (real_next_slice < file.size) {
493                    // Let's inform the user with the best upload % progress
494                    if (percent < percent_done) percent = percent_done;
495                    $('#dbi-upload-progress').html('Sending file (' + percent + '%)');
496
497                    //var progress_bar = document.getElementById('upload-progress-bar')
498                    //progress_bar.style.width = percent + '%';
499
500                    upload_file(real_next_slice); // More to upload, call function recursively
501                } else {
502                    // No more fragments to read
503                    // if all fragments have been sent we commit the transaction 
504                    goals++;
505                    if (goals == concurrency) {
506                        $('#dbi-upload-progress').html('Completing upload...');
507                        commit_uploaded_file();
508                    }
509                }
510            }
511        });
512    };
513    reader.readAsDataURL(blob);
514}
515
516function commit_uploaded_file() {
517
518    var formData = new FormData();
519
520    formData.append('fileUpload', new Blob(), file.name);
521    formData.append('fileType', file.type);
522    formData.append('sessionID', sessionID);
523
524    $.ajax({
525        url: dataStoreURL + 'upload.php',
526        type: 'POST',
527        dataType: 'json',
528        contentType: false,
529        cache: false,
530        processData: false,
531        data: formData,
532        error: function(data) {
533            $('#dbi-upload-progress').html('Upload failed.');
534            console.log('Upload failed: ' + data);
535        },
536        success: function(data) {
537            $('#dbi-upload-progress').html('Upload completed.');
538
539            //var progress_bar = document.getElementById('upload-progress-bar')
540            //progress_bar.style.width = '100%';
541
542            datasetID = data['dataset_id'];
543            md5 = null;
544            sessionID = null;
545
546            //console.log(data)
547
548            // Show correct URL in address bar.
549            addDatasetIDToURL(datasetID);
550
551            initModal();
552        },
553        complete: function() {
554            $('#dbi-upload-submit').removeAttr('disabled');
555        }
556    });
557}