@cryptotaxi247 / netdata-1 / commits / 6215c4d02

updated snapshot compression options

Costa Tsaousis (ktsaou) committed Nov 15, 2017 at 19:59 UTC 6215c4d020d4eefecddbb9b0dd9d469408bed781
1 file changed +192 -112
web/index.html
+192 -112
@@ -2870,6 +2870,117 @@
2870 });
2871 }
2872
2873 + // --------------------------------------------------------------------
2874 +
2875 + var snapshotOptions = {
2876 + bytes_per_chart: 2048,
2877 + compressionDefault: 'lzstring.utf16',
2878 +
2879 + compressions: {
2880 + 'none': {
2881 + bytes_per_point_memory: 5.2,
2882 + bytes_per_point_disk: 5.6,
2883 +
2884 + compress: function (s) {
2885 + return s;
2886 + },
2887 +
2888 + compressed_length: function (s) {
2889 + return s.length;
2890 + },
2891 +
2892 + uncompress: function (s) {
2893 + return s;
2894 + }
2895 + },
2896 +
2897 + 'pako.deflate.base64': {
2898 + bytes_per_point_memory: 1.8,
2899 + bytes_per_point_disk: 1.9,
2900 +
2901 + compress: function (s) {
2902 + return btoa(pako.deflate(s, { to: 'string' }));
2903 + },
2904 +
2905 + compressed_length: function (s) {
2906 + return s.length;
2907 + },
2908 +
2909 + uncompress: function (s) {
2910 + return pako.inflate(atob(s), {to: 'string'});
2911 + }
2912 + },
2913 +
2914 + 'pako.deflate': {
2915 + bytes_per_point_memory: 1.4,
2916 + bytes_per_point_disk: 3.2,
2917 +
2918 + compress: function (s) {
2919 + return pako.deflate(s, { to: 'string' });
2920 + },
2921 +
2922 + compressed_length: function (s) {
2923 + return s.length;
2924 + },
2925 +
2926 + uncompress: function (s) {
2927 + return pako.inflate(s, {to: 'string'});
2928 + }
2929 + },
2930 +
2931 + 'lzstring.utf16': {
2932 + bytes_per_point_memory: 1.7,
2933 + bytes_per_point_disk: 2.6,
2934 +
2935 + compress: function (s) {
2936 + return LZString.compressToUTF16(s);
2937 + },
2938 +
2939 + compressed_length: function (s) {
2940 + return s.length * 2;
2941 + },
2942 +
2943 + uncompress: function (s) {
2944 + return LZString.decompressFromUTF16(s);
2945 + }
2946 + },
2947 +
2948 + 'lzstring.base64': {
2949 + bytes_per_point_memory: 2.1,
2950 + bytes_per_point_disk: 2.3,
2951 +
2952 + compress: function (s) {
2953 + return LZString.compressToBase64(s);
2954 + },
2955 +
2956 + compressed_length: function (s) {
2957 + return s.length;
2958 + },
2959 +
2960 + uncompress: function (s) {
2961 + return LZString.decompressFromBase64(s);
2962 + }
2963 + },
2964 +
2965 + 'lzstring.uri': {
2966 + bytes_per_point_memory: 2.1,
2967 + bytes_per_point_disk: 2.3,
2968 +
2969 + compress: function (s) {
2970 + return LZString.compressToEncodedURIComponent(s);
2971 + },
2972 +
2973 + compressed_length: function (s) {
2974 + return s.length;
2975 + },
2976 +
2977 + uncompress: function (s) {
2978 + return LZString.decompressFromEncodedURIComponent(s);
2979 + }
2980 + }
2981 + }
2982 + };
2983 +
2984 // --------------------------------------------------------------------
2985 // loading snapshots
2986
@@ -2918,42 +3029,13 @@
3029 if (typeof tmpSnapshotData.compression !== 'string')
3030 tmpSnapshotData.compression = 'none';
3031
2921 - switch (tmpSnapshotData.compression) {
2922 - case 'pako.deflate.base64':
2923 - tmpSnapshotData.uncompress = function(s) {
2924 - return pako.inflate(atob(s), { to: 'string' });
2925 - };
2926 - break;
2927 -
2928 - case 'pako.deflate':
2929 - tmpSnapshotData.uncompress = function(s) {
2930 - return pako.inflate(s, { to: 'string' });
2931 - };
2932 - break;
2933 -
2934 - case 'lzstring.utf16':
2935 - tmpSnapshotData.uncompress = LZString.decompressFromUTF16;
2936 - break;
2937 -
2938 - case 'lzstring.base64':
2939 - tmpSnapshotData.uncompress = LZString.decompressFromBase64;
2940 - break;
2941 -
2942 - case 'lzstring.uri':
2943 - tmpSnapshotData.uncompress = LZString.decompressFromEncodedURIComponent;
2944 - break;
2945 -
2946 - case 'none':
2947 - tmpSnapshotData.uncompress = function (data) {
2948 - return data;
2949 - };
2950 - break;
2951 -
2952 - default:
2953 - alert('unknown compression method: ' + tmpSnapshotData.compression);
2954 - break;
3032 + if (typeof snapshotOptions.compressions[tmpSnapshotData.compression] === 'undefined') {
3033 + alert('unknown compression method: ' + tmpSnapshotData.compression);
3034 + tmpSnapshotData.compression = 'none';
3035 }
3036
3037 + tmpSnapshotData.uncompress = snapshotOptions.compressions[tmpSnapshotData.compression].uncompress;
3038 +
3039 netdataSnapshotData = tmpSnapshotData;
3040 initializeDynamicDashboard(netdataSnapshotData.url);
3041
@@ -3032,46 +3114,76 @@
3114 document.getElementById('saveSnapshotStatus').innerHTML = msg;
3115 }
3116
3035 - var saveSnapshotCompression = 'lzstring.utf16';
3117 + function saveSnapshotModalShowExpectedSize() {
3118 + var points = Math.round(saveSnapshotViewDuration / saveSnapshotSelectedSecondsPerPoint);
3119 + var priority = 'info';
3120 + var msg = 'A moderate snapshot.';
3121 +
3122 + var sizemb = Math.round(
3123 + (options.data.charts_count * snapshotOptions.bytes_per_chart
3124 + + options.data.dimensions_count * points * snapshotOptions.compressions[saveSnapshotCompression].bytes_per_point_disk)
3125 + * 10 / 1024 / 1024) / 10;
3126 +
3127 + var memmb = Math.round(
3128 + (options.data.charts_count * snapshotOptions.bytes_per_chart
3129 + + options.data.dimensions_count * points * snapshotOptions.compressions[saveSnapshotCompression].bytes_per_point_memory)
3130 + * 10 / 1024 / 1024) / 10;
3131 +
3132 + if(sizemb < 10) {
3133 + priority = 'success';
3134 + msg = 'A nice small snapshot!';
3135 + }
3136 + if(sizemb > 50) {
3137 + priority = 'warning';
3138 + msg = 'Will stress your browser...';
3139 + }
3140 + if(sizemb > 100) {
3141 + priority = 'danger';
3142 + msg = 'Hm... good luck...';
3143 + }
3144 +
3145 + saveSnapshotModalLog(priority, 'The snapshot will have ' + points.toString() + ' points per dimension. Expected size on disk ' + sizemb + ' MB, on browser memory ' + memmb + ' MB.<br/>' + msg);
3146 + }
3147 +
3148 + var saveSnapshotCompression = snapshotOptions.compressionDefault;
3149 function saveSnapshotSetCompression(name) {
3150 saveSnapshotCompression = name;
3151 document.getElementById('saveSnapshotCompressionName').innerHTML = saveSnapshotCompression;
3152 + saveSnapshotModalShowExpectedSize();
3153 }
3154
3155 var saveSnapshotSlider = null;
3156 var saveSnapshotSelectedSecondsPerPoint = 1;
3157 + var saveSnapshotViewDuration = 1;
3158 function saveSnapshotModalInit() {
3159 $('#saveSnapshotModalProgressSection').hide();
3160 $('#saveSnapshotResolutionRadio').show();
3161 saveSnapshotModalLog('info', 'Select resolution and click <b>Save</b>');
3162
3163 loadBootstrapSlider(function() {
3049 - var view_duration = options.duration;
3050 - var start_ms = Math.round(Date.now() - view_duration * 1000);
3051 -
3052 - var bytes_per_chart = 2048;
3053 - var bytes_per_point_per_dimension = 2.6;
3164 + saveSnapshotViewDuration = options.duration;
3165 + var start_ms = Math.round(Date.now() - saveSnapshotViewDuration * 1000);
3166
3167 if(NETDATA.globalPanAndZoom.isActive() === true) {
3056 - view_duration = Math.round((NETDATA.globalPanAndZoom.force_before_ms - NETDATA.globalPanAndZoom.force_after_ms) / 1000);
3168 + saveSnapshotViewDuration = Math.round((NETDATA.globalPanAndZoom.force_before_ms - NETDATA.globalPanAndZoom.force_after_ms) / 1000);
3169 start_ms = NETDATA.globalPanAndZoom.force_after_ms;
3170 }
3171
3172 var start_date = new Date(start_ms);
3173 var yyyymmddhhssmm = start_date.getFullYear() + NETDATA.zeropad(start_date.getMonth() + 1) + NETDATA.zeropad(start_date.getDate()) + '-' + NETDATA.zeropad(start_date.getHours()) + NETDATA.zeropad(start_date.getMinutes()) + NETDATA.zeropad(start_date.getSeconds());
3174
3063 - document.getElementById('saveSnapshotFilename').value = 'netdata-' + options.hostname.toString() + '-' + yyyymmddhhssmm.toString() + '-' + view_duration.toString() + '.snapshot'
3175 + document.getElementById('saveSnapshotFilename').value = 'netdata-' + options.hostname.toString() + '-' + yyyymmddhhssmm.toString() + '-' + saveSnapshotViewDuration.toString() + '.snapshot'
3176 saveSnapshotSetCompression(saveSnapshotCompression);
3177
3178 var min = options.update_every;
3067 - var max = Math.round(view_duration / 100);
3179 + var max = Math.round(saveSnapshotViewDuration / 100);
3180
3181 if(NETDATA.globalPanAndZoom.isActive() === false)
3070 - max = Math.round(view_duration / 50);
3182 + max = Math.round(saveSnapshotViewDuration / 50);
3183
3072 - var view = Math.round(view_duration / Math.round($(document.getElementById('charts_div')).width() / 2));
3184 + var view = Math.round(saveSnapshotViewDuration / Math.round($(document.getElementById('charts_div')).width() / 2));
3185
3074 - // console.log('view duration: ' + view_duration + ', min: ' + min + ', max: ' + max + ', view: ' + view);
3186 + // console.log('view duration: ' + saveSnapshotViewDuration + ', min: ' + min + ', max: ' + max + ', view: ' + view);
3187
3188 if(max < 10) max = 10;
3189 if(max < min) max = min;
@@ -3090,28 +3202,20 @@
3202 scale: (max > 100)?'logarithmic':'linear',
3203 tooltip: 'always',
3204 formatter: function(value) {
3093 - var points = Math.round(view_duration / value);
3094 - var priority = 'info';
3095 - var msg = 'A moderate snapshot.';
3096 -
3097 - var sizemb = Math.round((options.data.charts_count * bytes_per_chart + options.data.dimensions_count * points * bytes_per_point_per_dimension) / 1024 / 1024);
3205 + if(value < 1)
3206 + value = 1;
3207
3099 - if(sizemb < 10) {
3100 - priority = 'success';
3101 - msg = 'A nice small snapshot!';
3102 - }
3103 - if(sizemb > 50) {
3104 - priority = 'warning';
3105 - msg = 'Will stress your browser...';
3106 - }
3107 - if(sizemb > 100) {
3108 - priority = 'danger';
3109 - msg = 'Hm... good luck...';
3110 - }
3208 + if(value < options.data.update_every)
3209 + value = options.data.update_every;
3210
3112 - saveSnapshotModalLog(priority, 'The snapshot will have ' + points.toString() + ' points per dimension. Expected size ' + sizemb + ' MB. ' + msg);
3211 saveSnapshotSelectedSecondsPerPoint = value;
3114 - return value + ' seconds per point';
3212 + saveSnapshotModalShowExpectedSize();
3213 +
3214 + var seconds = ' seconds ';
3215 + if(value === 1)
3216 + seconds = ' second ';
3217 +
3218 + return value + seconds + 'per point' + ((value === options.data.update_every)?', server default':'').toString();
3219 }
3220 });
3221 });
@@ -3179,46 +3283,14 @@
3283 data: {}
3284 };
3285
3182 - var compress_length_multiplier = 1;
3183 - var compress;
3184 - switch (saveData.compression) {
3185 - case 'pako.deflate.base64':
3186 - compress = function(s) {
3187 - return btoa(pako.deflate(s, { to: 'string' }));
3188 - };
3189 - compress_length_multiplier = 1;
3190 - break;
3191 -
3192 - case 'pako.deflate':
3193 - compress = function(s) {
3194 - return pako.deflate(s, { to: 'string' });
3195 - };
3196 - compress_length_multiplier = 1;
3197 - break;
3198 -
3199 - case 'lzstring.utf16':
3200 - compress = LZString.compressToUTF16;
3201 - compress_length_multiplier = 2;
3202 - break;
3203 -
3204 - case 'lzstring.base64':
3205 - compress = LZString.compressToBase64;
3206 - compress_length_multiplier = 1;
3207 - break;
3208 -
3209 - default:
3210 - alert('unknown compression method: ' + saveData.compression);
3211 - saveData.compression = 'none';
3212 - // no break here, let it continue
3213 -
3214 - case 'none':
3215 - compress = function (data) {
3216 - return data;
3217 - };
3218 - compress_length_multiplier = 1;
3219 - break;
3286 + if(typeof snapshotOptions.compressions[saveData.compression] === 'undefined') {
3287 + alert('unknown compression method: ' + saveData.compression);
3288 + saveData.compression = 'none';
3289 }
3290
3291 + var compress = snapshotOptions.compressions[saveData.compression].compress;
3292 + var compressed_length = snapshotOptions.compressions[saveData.compression].compressed_length;
3293 +
3294 function pack_api1_v1_chart_data(state) {
3295 if (state.library_name === null || state.data === null)
3296 return;
@@ -3231,7 +3303,7 @@
3303 if (typeof str === 'string') {
3304 var cstr = compress(str);
3305 saveData.data[state.chartDataUniqueID()] = cstr;
3234 - return cstr.length * compress_length_multiplier;
3306 + return compressed_length(cstr);
3307 }
3308 else
3309 return 0;
@@ -3323,6 +3395,7 @@
3395 saveData.charts_ok = charts_ok;
3396 saveData.charts_failed = charts_failed;
3397 saveData.data_size = size;
3398 + // console.log(saveData.compression + ': ' + (size / (options.data.dimensions_count * Math.round(saveSnapshotViewDuration / saveSnapshotSelectedSecondsPerPoint))).toString());
3399
3400 // save it
3401 // console.log(saveData);
@@ -4094,13 +4167,13 @@
4167 </p>
4168 <p>
4169 <label class="btn btn-default">
4097 - Select a netdata saved JSON file to load
4170 + Click here to select a netdata snapshot file
4171 <input type="file" id="loadSnapshotSelectFiles" value="Import" style="display: none;"
4172 onchange="loadSnapshotPreflight();">
4173 </label>
4174 </p>
4175 <div id="loadSnapshotStatus" class="alert alert-info" role="alert">
4103 - Click the button above to select the file to load, then click <b>Import</b> to render it.
4176 + Open a snapshot file, then click <b>Import</b> to render it.
4177 </div>
4178 <p>
4179 <table class="table">
@@ -4129,7 +4202,7 @@
4202 <div class="modal-content">
4203 <div class="modal-header">
4204 <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
4132 - <h4 class="modal-title" id="saveSnapshotModalLabel">Export a dashboard snapshot</h4>
4205 + <h4 class="modal-title" id="saveSnapshotModalLabel">Export a snapshot</h4>
4206 </div>
4207 <div class="modal-body">
4208 <div id="saveSnapshotModalProgressSection" hidden>
@@ -4159,11 +4232,12 @@
4232 <ul class="dropdown-menu dropdown-menu-right">
4233 <li class="disabled"><a href="#" class="disabled">Select Compression</a></li>
4234 <li role="separator" class="divider"></li>
4162 - <li><a href="#" onclick="saveSnapshotSetCompression('none'); return false;">No Compression</a></li>
4235 + <li><a href="#" onclick="saveSnapshotSetCompression('none'); return false;">uncompressed</a></li>
4236 <li role="separator" class="divider"></li>
4237 <li><a href="#" onclick="saveSnapshotSetCompression('pako.deflate'); return false;">pako.deflate (gzip, binary)</a></li>
4238 <li><a href="#" onclick="saveSnapshotSetCompression('pako.deflate.base64'); return false;">pako.deflate.base64 (gzip, ascii)</a></li>
4239 <li role="separator" class="divider"></li>
4240 + <li><a href="#" onclick="saveSnapshotSetCompression('lzstring.uri'); return false;">lzstring.uri (LZ, ascii)</a></li>
4241 <li><a href="#" onclick="saveSnapshotSetCompression('lzstring.utf16'); return false;">lzstring.utf16 (LZ, utf16)</a></li>
4242 <li><a href="#" onclick="saveSnapshotSetCompression('lzstring.base64'); return false;">lzstring.base64 (LZ, ascii)</a></li>
4243 </ul>
@@ -4178,14 +4252,20 @@
4252
4253 &nbsp;
4254 <div id="saveSnapshotStatus" class="alert alert-info" role="alert">
4181 - Select resolution for saving the snapshot.
4255 + Select snaphost resolution. This controls the size the snapshot file.
4256 </div>
4257 <p>
4184 - The generated snapshot will include all charts of this dashboard, <b>for the visible timeframe</b>, so align, pan and zoom the charts as needed.
4185 - The snapshot will be downloaded on your computer, as a JSON file, that can be loaded back into any netdata dashboard (no need to load it on this one).
4258 + The generated snapshot will include all charts of this dashboard, <b>for the visible timeframe</b>, so align, pan and zoom the charts as needed.
4259 + The scroll position of the dashboard will also be saved.
4260 + The snapshot will be downloaded as a file, to your computer, that can be imported back into any netdata dashboard (no need to import it back on this server).
4261 </p>
4262 <p>
4188 - Snapshots are handled entirely by the web browser. The netdata servers are not aware of them.
4263 + <small>
4264 + Snapshot files include all the information of the dashboard, including the URL of the origin server, its netdata unique ID, etc.
4265 + So, if you share the snapshot file with third parties, they will be able to access the origin server, if this server is exposed on the internet.
4266 + <br/>
4267 + Snapshots are handled entirely by the web browser. The netdata servers are not aware of them.
4268 + </small>
4269 </p>
4270 </div>
4271 <div class="modal-footer">