@cryptotaxi247 / netdata-1 / commits / 9b4f14914

global statistics now also report max API response time; global statistics implemented with alternative lock-free code; more code cleanups

Costa Tsaousis committed Aug 13, 2016 at 01:44 UTC 9b4f149140248792bf2ccbc6e6f90222e5e078ab
12 files changed +360 -228
src/apps_plugin.c
+7 -7
@@ -642,7 +642,7 @@ int read_proc_pid_stat(struct pid_stat *p) {
642 if(unlikely(!ff)) goto cleanup;
643
644 p->last_stat_collected_usec = p->stat_collected_usec;
645 - p->stat_collected_usec = timems();
645 + p->stat_collected_usec = time_usec();
646 file_counter++;
647
648 // p->pid = atol(procfile_lineword(ff, 0, 0+i));
@@ -824,7 +824,7 @@ int read_proc_pid_io(struct pid_stat *p) {
824 file_counter++;
825
826 p->last_io_collected_usec = p->io_collected_usec;
827 - p->io_collected_usec = timems();
827 + p->io_collected_usec = time_usec();
828
829 unsigned long long last;
830
@@ -898,7 +898,7 @@ int read_proc_stat() {
898 if(unlikely(!ff)) goto cleanup;
899
900 last_collected_usec = collected_usec;
901 - collected_usec = timems();
901 + collected_usec = time_usec();
902
903 file_counter++;
904
@@ -2205,7 +2205,7 @@ unsigned long long send_resource_usage_to_netdata() {
2205 gettimeofday(&now, NULL);
2206 getrusage(RUSAGE_SELF, &me);
2207
2208 - usec = usecdiff(&now, &last);
2208 + usec = usec_dt(&now, &last);
2209 cpuuser = me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec;
2210 cpusyst = me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec;
2211
@@ -2836,11 +2836,11 @@ int main(int argc, char **argv)
2836 for(;1; global_iterations_counter++) {
2837 #ifndef PROFILING_MODE
2838 // delay until it is our time to run
2839 - while((sunow = timems()) < sunext)
2840 - usecsleep(sunext - sunow);
2839 + while((sunow = time_usec()) < sunext)
2840 + sleep_usec(sunext - sunow);
2841
2842 // find the next time we need to run
2843 - while(timems() > sunext)
2843 + while(time_usec() > sunext)
2844 sunext += update_every * 1000000ULL;
2845 #endif /* PROFILING_MODE */
2846
src/common.c
+15 -4
@@ -43,15 +43,26 @@ void freez(void *ptr) {
43 }
44
45 // ----------------------------------------------------------------------------
46 +// time functions
47
47 -// time(NULL) in milliseconds
48 -unsigned long long timems(void) {
48 +inline unsigned long long timeval_usec(struct timeval *tv) {
49 + return tv->tv_sec * 1000000ULL + tv->tv_usec;
50 +}
51 +
52 +// time(NULL) in nanoseconds
53 +inline unsigned long long time_usec(void) {
54 struct timeval now;
55 gettimeofday(&now, NULL);
51 - return now.tv_sec * 1000000ULL + now.tv_usec;
56 + return timeval_usec(&now);
57 +}
58 +
59 +inline unsigned long long usec_dt(struct timeval *now, struct timeval *old) {
60 + unsigned long long tv1 = timeval_usec(now);
61 + unsigned long long tv2 = timeval_usec(old);
62 + return (tv1 > tv2)?(tv1 - tv2):(tv2 - tv1);
63 }
64
54 -int usecsleep(unsigned long long usec) {
65 +int sleep_usec(unsigned long long usec) {
66
67 #ifndef NETDATA_WITH_USLEEP
68 // we expect microseconds (1.000.000 per second)
src/common.h
+6 -3
@@ -107,7 +107,10 @@
107 #endif
108 #define abs(x) ((x < 0)? -x : x)
109
110 -#define usecdiff(now, last) (((((now)->tv_sec * 1000000ULL) + (now)->tv_usec) - (((last)->tv_sec * 1000000ULL) + (last)->tv_usec)))
110 +extern unsigned long long usec_dt(struct timeval *now, struct timeval *old);
111 +extern unsigned long long timeval_usec(struct timeval *tv);
112 +
113 +// #define usec_dt(now, last) (((((now)->tv_sec * 1000000ULL) + (now)->tv_usec) - (((last)->tv_sec * 1000000ULL) + (last)->tv_usec)))
114
115 extern void netdata_fix_chart_id(char *s);
116 extern void netdata_fix_chart_name(char *s);
@@ -144,8 +147,8 @@ extern void get_HZ(void);
147
148 extern pid_t gettid(void);
149
147 -extern unsigned long long timems(void);
148 -extern int usecsleep(unsigned long long usec);
150 +extern unsigned long long time_usec(void);
151 +extern int sleep_usec(unsigned long long usec);
152
153 extern char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len);
154
src/global_statistics.c
+254 -5
@@ -1,13 +1,262 @@
1 #include "common.h"
2
3 -struct global_statistics global_statistics = { 0, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL};
3 +volatile struct global_statistics global_statistics = {
4 + .connected_clients = 0,
5 + .web_requests = 0,
6 + .web_usec = 0,
7 + .bytes_received = 0,
8 + .bytes_sent = 0,
9 + .content_size = 0,
10 + .compressed_content_size = 0
11 +};
12
13 pthread_mutex_t global_statistics_mutex = PTHREAD_MUTEX_INITIALIZER;
14
7 -void global_statistics_lock(void) {
8 - pthread_mutex_lock(&global_statistics_mutex);
15 +inline void global_statistics_lock(void) {
16 + pthread_mutex_lock(&global_statistics_mutex);
17 }
18
11 -void global_statistics_unlock(void) {
12 - pthread_mutex_unlock(&global_statistics_mutex);
19 +inline void global_statistics_unlock(void) {
20 + pthread_mutex_unlock(&global_statistics_mutex);
21 }
22 +
23 +void finished_web_request_statistics(uint64_t dt,
24 + uint64_t bytes_received,
25 + uint64_t bytes_sent,
26 + uint64_t content_size,
27 + uint64_t compressed_content_size) {
28 +#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS
29 + uint64_t old_web_usec_max = global_statistics.web_usec_max;
30 + while(dt > old_web_usec_max) {
31 + if(__sync_bool_compare_and_swap(&global_statistics.web_usec_max, old_web_usec_max, dt))
32 + old_web_usec_max = dt;
33 + else
34 + old_web_usec_max = global_statistics.web_usec_max;
35 + }
36 +
37 + __sync_fetch_and_add(&global_statistics.web_requests, 1);
38 + __sync_fetch_and_add(&global_statistics.web_usec, dt);
39 + __sync_fetch_and_add(&global_statistics.bytes_received, bytes_received);
40 + __sync_fetch_and_add(&global_statistics.bytes_sent, bytes_sent);
41 + __sync_fetch_and_add(&global_statistics.content_size, content_size);
42 + __sync_fetch_and_add(&global_statistics.compressed_content_size, compressed_content_size);
43 +#else
44 + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
45 + global_statistics_lock();
46 +
47 + if (dt > global_statistics.web_usec_max)
48 + global_statistics.web_usec_max = dt;
49 +
50 + global_statistics.web_requests++;
51 + global_statistics.web_usec += dt;
52 + global_statistics.bytes_received += bytes_received;
53 + global_statistics.bytes_sent += bytes_sent;
54 + global_statistics.content_size += content_size;
55 + global_statistics.compressed_content_size += compressed_content_size;
56 +
57 + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
58 + global_statistics_unlock();
59 +#endif
60 +}
61 +
62 +void web_client_connected(void) {
63 +#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS
64 + __sync_fetch_and_add(&global_statistics.connected_clients, 1);
65 +#else
66 + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
67 + global_statistics_lock();
68 +
69 + global_statistics.connected_clients++;
70 +
71 + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
72 + global_statistics_unlock();
73 +#endif
74 +}
75 +
76 +void web_client_disconnected(void) {
77 +#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS
78 + __sync_fetch_and_sub(&global_statistics.connected_clients, 1);
79 +#else
80 + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
81 + global_statistics_lock();
82 +
83 + global_statistics.connected_clients--;
84 +
85 + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
86 + global_statistics_unlock();
87 +#endif
88 +}
89 +
90 +
91 +inline void global_statistics_copy(struct global_statistics *gs, uint8_t options) {
92 +#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS
93 + gs->connected_clients = __sync_fetch_and_add(&global_statistics.connected_clients, 0);
94 + gs->web_requests = __sync_fetch_and_add(&global_statistics.web_requests, 0);
95 + gs->web_usec = __sync_fetch_and_add(&global_statistics.web_usec, 0);
96 + gs->web_usec_max = __sync_fetch_and_add(&global_statistics.web_usec_max, 0);
97 + gs->bytes_received = __sync_fetch_and_add(&global_statistics.bytes_received, 0);
98 + gs->bytes_sent = __sync_fetch_and_add(&global_statistics.bytes_sent, 0);
99 + gs->content_size = __sync_fetch_and_add(&global_statistics.content_size, 0);
100 + gs->compressed_content_size = __sync_fetch_and_add(&global_statistics.compressed_content_size, 0);
101 +
102 + if(options & GLOBAL_STATS_RESET_WEB_USEC_MAX)
103 + __sync_bool_compare_and_swap(&global_statistics.web_usec_max, gs->web_usec_max, 0);
104 +#else
105 + global_statistics_lock();
106 +
107 + memcpy(gs, (const void *)&global_statistics, sizeof(struct global_statistics));
108 +
109 + if (options & GLOBAL_STATS_RESET_WEB_USEC_MAX)
110 + global_statistics.web_usec_max = 0;
111 +
112 + global_statistics_unlock();
113 +#endif
114 +}
115 +
116 +void global_statistics_charts(void) {
117 + static unsigned long long old_web_requests = 0, old_web_usec = 0,
118 + old_content_size = 0, old_compressed_content_size = 0;
119 +
120 + static collected_number compression_ratio = -1, average_response_time = -1;
121 +
122 + static RRDSET *stcpu = NULL, *stcpu_thread = NULL, *stclients = NULL, *streqs = NULL, *stbytes = NULL, *stduration = NULL,
123 + *stcompression = NULL;
124 +
125 + struct global_statistics gs;
126 + struct rusage me, thread;
127 +
128 + global_statistics_copy(&gs, GLOBAL_STATS_RESET_WEB_USEC_MAX);
129 + getrusage(RUSAGE_THREAD, &thread);
130 + getrusage(RUSAGE_SELF, &me);
131 +
132 + if (!stcpu_thread) stcpu_thread = rrdset_find("netdata.plugin_proc_cpu");
133 + if (!stcpu_thread) {
134 + stcpu_thread = rrdset_create("netdata", "plugin_proc_cpu", NULL, "proc.internal", NULL,
135 + "NetData Proc Plugin CPU usage", "milliseconds/s", 132000, rrd_update_every,
136 + RRDSET_TYPE_STACKED);
137 +
138 + rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL);
139 + rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL);
140 + } else rrdset_next(stcpu_thread);
141 +
142 + rrddim_set(stcpu_thread, "user", thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
143 + rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
144 + rrdset_done(stcpu_thread);
145 +
146 + // ----------------------------------------------------------------
147 +
148 + if (!stcpu) stcpu = rrdset_find("netdata.server_cpu");
149 + if (!stcpu) {
150 + stcpu = rrdset_create("netdata", "server_cpu", NULL, "netdata", NULL, "NetData CPU usage", "milliseconds/s",
151 + 130000, rrd_update_every, RRDSET_TYPE_STACKED);
152 +
153 + rrddim_add(stcpu, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL);
154 + rrddim_add(stcpu, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL);
155 + } else rrdset_next(stcpu);
156 +
157 + rrddim_set(stcpu, "user", me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec);
158 + rrddim_set(stcpu, "system", me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec);
159 + rrdset_done(stcpu);
160 +
161 + // ----------------------------------------------------------------
162 +
163 + if (!stclients) stclients = rrdset_find("netdata.clients");
164 + if (!stclients) {
165 + stclients = rrdset_create("netdata", "clients", NULL, "netdata", NULL, "NetData Web Clients",
166 + "connected clients", 130100, rrd_update_every, RRDSET_TYPE_LINE);
167 +
168 + rrddim_add(stclients, "clients", NULL, 1, 1, RRDDIM_ABSOLUTE);
169 + } else rrdset_next(stclients);
170 +
171 + rrddim_set(stclients, "clients", gs.connected_clients);
172 + rrdset_done(stclients);
173 +
174 + // ----------------------------------------------------------------
175 +
176 + if (!streqs) streqs = rrdset_find("netdata.requests");
177 + if (!streqs) {
178 + streqs = rrdset_create("netdata", "requests", NULL, "netdata", NULL, "NetData Web Requests", "requests/s",
179 + 130200, rrd_update_every, RRDSET_TYPE_LINE);
180 +
181 + rrddim_add(streqs, "requests", NULL, 1, 1, RRDDIM_INCREMENTAL);
182 + } else rrdset_next(streqs);
183 +
184 + rrddim_set(streqs, "requests", (collected_number) gs.web_requests);
185 + rrdset_done(streqs);
186 +
187 + // ----------------------------------------------------------------
188 +
189 + if (!stbytes) stbytes = rrdset_find("netdata.net");
190 + if (!stbytes) {
191 + stbytes = rrdset_create("netdata", "net", NULL, "netdata", NULL, "NetData Network Traffic", "kilobits/s",
192 + 130300, rrd_update_every, RRDSET_TYPE_AREA);
193 +
194 + rrddim_add(stbytes, "in", NULL, 8, 1024, RRDDIM_INCREMENTAL);
195 + rrddim_add(stbytes, "out", NULL, -8, 1024, RRDDIM_INCREMENTAL);
196 + } else rrdset_next(stbytes);
197 +
198 + rrddim_set(stbytes, "in", (collected_number) gs.bytes_received);
199 + rrddim_set(stbytes, "out", (collected_number) gs.bytes_sent);
200 + rrdset_done(stbytes);
201 +
202 + // ----------------------------------------------------------------
203 +
204 + if (!stduration) stduration = rrdset_find("netdata.response_time");
205 + if (!stduration) {
206 + stduration = rrdset_create("netdata", "response_time", NULL, "netdata", NULL, "NetData API Response Time",
207 + "ms/request", 130400, rrd_update_every, RRDSET_TYPE_LINE);
208 +
209 + rrddim_add(stduration, "average", NULL, 1, 1000, RRDDIM_ABSOLUTE);
210 + rrddim_add(stduration, "max", NULL, 1, 1000, RRDDIM_ABSOLUTE);
211 + } else rrdset_next(stduration);
212 +
213 + uint64_t gweb_usec = gs.web_usec;
214 + uint64_t gweb_requests = gs.web_requests;
215 +
216 + uint64_t web_usec = (gweb_usec >= old_web_usec) ? gweb_usec - old_web_usec : 0;
217 + uint64_t web_requests = (gweb_requests >= old_web_requests) ? gweb_requests - old_web_requests : 0;
218 +
219 + old_web_usec = gweb_usec;
220 + old_web_requests = gweb_requests;
221 +
222 + if (web_requests)
223 + average_response_time = (collected_number) (web_usec / web_requests);
224 +
225 + if (unlikely(average_response_time != -1))
226 + rrddim_set(stduration, "average", average_response_time);
227 + else
228 + rrddim_set(stduration, "average", 0);
229 +
230 + rrddim_set(stduration, "max", ((gs.web_usec_max)?(collected_number)gs.web_usec_max:average_response_time));
231 + rrdset_done(stduration);
232 +
233 + // ----------------------------------------------------------------
234 +
235 + if (!stcompression) stcompression = rrdset_find("netdata.compression_ratio");
236 + if (!stcompression) {
237 + stcompression = rrdset_create("netdata", "compression_ratio", NULL, "netdata", NULL,
238 + "NetData API Responses Compression Savings Ratio", "percentage", 130500,
239 + rrd_update_every, RRDSET_TYPE_LINE);
240 +
241 + rrddim_add(stcompression, "savings", NULL, 1, 1000, RRDDIM_ABSOLUTE);
242 + } else rrdset_next(stcompression);
243 +
244 + // since we don't lock here to read the global statistics
245 + // read the smaller value first
246 + unsigned long long gcompressed_content_size = gs.compressed_content_size;
247 + unsigned long long gcontent_size = gs.content_size;
248 +
249 + unsigned long long compressed_content_size = gcompressed_content_size - old_compressed_content_size;
250 + unsigned long long content_size = gcontent_size - old_content_size;
251 +
252 + old_compressed_content_size = gcompressed_content_size;
253 + old_content_size = gcontent_size;
254 +
255 + if (content_size && content_size >= compressed_content_size)
256 + compression_ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size;
257 +
258 + if (compression_ratio != -1)
259 + rrddim_set(stcompression, "savings", compression_ratio);
260 +
261 + rrdset_done(stcompression);
262 +}
\ No newline at end of file
src/global_statistics.h
+22 -8
@@ -5,18 +5,32 @@
5 // global statistics
6
7 struct global_statistics {
8 - volatile unsigned long volatile connected_clients;
9 - volatile unsigned long long volatile web_requests;
10 - volatile unsigned long long volatile web_usec;
11 - volatile unsigned long long volatile bytes_received;
12 - volatile unsigned long long volatile bytes_sent;
13 - volatile unsigned long long volatile content_size;
14 - volatile unsigned long long volatile compressed_content_size;
8 + volatile uint16_t connected_clients;
9 +
10 + volatile uint64_t web_requests;
11 + volatile uint64_t web_usec;
12 + volatile uint64_t web_usec_max;
13 + volatile uint64_t bytes_received;
14 + volatile uint64_t bytes_sent;
15 + volatile uint64_t content_size;
16 + volatile uint64_t compressed_content_size;
17 };
18
17 -extern struct global_statistics global_statistics;
19 +extern volatile struct global_statistics global_statistics;
20
21 extern void global_statistics_lock(void);
22 extern void global_statistics_unlock(void);
23 +extern void finished_web_request_statistics(uint64_t dt,
24 + uint64_t bytes_received,
25 + uint64_t bytes_sent,
26 + uint64_t content_size,
27 + uint64_t compressed_content_size);
28 +
29 +extern void web_client_connected(void);
30 +extern void web_client_disconnected(void);
31 +
32 +#define GLOBAL_STATS_RESET_WEB_USEC_MAX 0x01
33 +extern void global_statistics_copy(struct global_statistics *gs, uint8_t options);
34 +extern void global_statistics_charts(void);
35
36 #endif /* NETDATA_GLOBAL_STATISTICS_H */
src/plugin_checks.c
+4 -4
@@ -36,7 +36,7 @@ void *checks_main(void *ptr)
36
37 // find the time to sleep in order to wait exactly update_every seconds
38 gettimeofday(&now, NULL);
39 - loop_usec = usecdiff(&now, &last);
39 + loop_usec = usec_dt(&now, &last);
40 usec = loop_usec - susec;
41 debug(D_PROCNETDEV_LOOP, "CHECK: last loop took %llu usec (worked for %llu, sleeped for %llu).", loop_usec, usec, susec);
42
@@ -72,9 +72,9 @@ void *checks_main(void *ptr)
72 if(!apps_cpu) apps_cpu = rrdset_find("apps.cpu");
73 if(check3->counter_done) rrdset_next_usec(check3, loop_usec);
74 gettimeofday(&loop, NULL);
75 - rrddim_set(check3, "caller", (long long)usecdiff(&loop, &check1->last_collected_time));
76 - rrddim_set(check3, "netdata", (long long)usecdiff(&loop, &check2->last_collected_time));
77 - if(apps_cpu) rrddim_set(check3, "apps.plugin", (long long)usecdiff(&loop, &apps_cpu->last_collected_time));
75 + rrddim_set(check3, "caller", (long long) usec_dt(&loop, &check1->last_collected_time));
76 + rrddim_set(check3, "netdata", (long long) usec_dt(&loop, &check2->last_collected_time));
77 + if(apps_cpu) rrddim_set(check3, "apps.plugin", (long long) usec_dt(&loop, &apps_cpu->last_collected_time));
78 rrdset_done(check3);
79 }
80
src/plugin_idlejitter.c
+1 -1
@@ -38,7 +38,7 @@ void *cpuidlejitter_main(void *ptr)
38 gettimeofday(&after, NULL);
39
40 // calculate the time it took for a full loop
41 - usec = usecdiff(&after, &before);
41 + usec = usec_dt(&after, &before);
42 susec += usec;
43 }
44 usec -= (sleep_ms * 1000);
src/plugin_proc.c
+21 -160
@@ -4,11 +4,6 @@ void *proc_main(void *ptr)
4 {
5 (void)ptr;
6
7 - unsigned long long old_web_requests = 0, old_web_usec = 0,
8 - old_content_size = 0, old_compressed_content_size = 0;
9 -
10 - collected_number compression_ratio = -1, average_response_time = -1;
11 -
7 info("PROC Plugin thread created with task id %d", gettid());
8
9 if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
@@ -17,8 +12,6 @@ void *proc_main(void *ptr)
12 if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
13 error("Cannot set pthread cancel state to ENABLE.");
14
20 - struct rusage me, thread;
21 -
15 // disable (by default) various interface that are not needed
16 config_get_boolean("plugin:proc:/proc/net/dev:lo", "enabled", 0);
17 config_get_boolean("plugin:proc:/proc/net/dev:fireqos_monitor", "enabled", 0);
@@ -66,18 +59,15 @@ void *proc_main(void *ptr)
59 unsigned long long sunext = (time(NULL) - (time(NULL) % rrd_update_every) + rrd_update_every) * 1000000ULL;
60 unsigned long long sunow;
61
69 - RRDSET *stcpu = NULL, *stcpu_thread = NULL, *stclients = NULL, *streqs = NULL, *stbytes = NULL, *stduration = NULL,
70 - *stcompression = NULL;
71 -
62 for(;1;) {
63 if(unlikely(netdata_exit)) break;
64
65 // delay until it is our time to run
76 - while((sunow = timems()) < sunext)
77 - usecsleep(sunext - sunow);
66 + while((sunow = time_usec()) < sunext)
67 + sleep_usec(sunext - sunow);
68
69 // find the next time we need to run
80 - while(timems() > sunext)
70 + while(time_usec() > sunext)
71 sunext += rrd_update_every * 1000000ULL;
72
73 if(unlikely(netdata_exit)) break;
@@ -87,7 +77,7 @@ void *proc_main(void *ptr)
77 if(!vdo_sys_kernel_mm_ksm) {
78 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_sys_kernel_mm_ksm().");
79
90 - sunow = timems();
80 + sunow = time_usec();
81 vdo_sys_kernel_mm_ksm = do_sys_kernel_mm_ksm(rrd_update_every, (sutime_sys_kernel_mm_ksm > 0)?sunow - sutime_sys_kernel_mm_ksm:0ULL);
82 sutime_sys_kernel_mm_ksm = sunow;
83 }
@@ -95,7 +85,7 @@ void *proc_main(void *ptr)
85
86 if(!vdo_proc_loadavg) {
87 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_loadavg().");
98 - sunow = timems();
88 + sunow = time_usec();
89 vdo_proc_loadavg = do_proc_loadavg(rrd_update_every, (sutime_proc_loadavg > 0)?sunow - sutime_proc_loadavg:0ULL);
90 sutime_proc_loadavg = sunow;
91 }
@@ -103,7 +93,7 @@ void *proc_main(void *ptr)
93
94 if(!vdo_proc_interrupts) {
95 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_interrupts().");
106 - sunow = timems();
96 + sunow = time_usec();
97 vdo_proc_interrupts = do_proc_interrupts(rrd_update_every, (sutime_proc_interrupts > 0)?sunow - sutime_proc_interrupts:0ULL);
98 sutime_proc_interrupts = sunow;
99 }
@@ -111,7 +101,7 @@ void *proc_main(void *ptr)
101
102 if(!vdo_proc_softirqs) {
103 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_softirqs().");
114 - sunow = timems();
104 + sunow = time_usec();
105 vdo_proc_softirqs = do_proc_softirqs(rrd_update_every, (sutime_proc_softirqs > 0)?sunow - sutime_proc_softirqs:0ULL);
106 sutime_proc_softirqs = sunow;
107 }
@@ -119,7 +109,7 @@ void *proc_main(void *ptr)
109
110 if(!vdo_proc_sys_kernel_random_entropy_avail) {
111 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_sys_kernel_random_entropy_avail().");
122 - sunow = timems();
112 + sunow = time_usec();
113 vdo_proc_sys_kernel_random_entropy_avail = do_proc_sys_kernel_random_entropy_avail(rrd_update_every, (sutime_proc_sys_kernel_random_entropy_avail > 0)?sunow - sutime_proc_sys_kernel_random_entropy_avail:0ULL);
114 sutime_proc_sys_kernel_random_entropy_avail = sunow;
115 }
@@ -127,7 +117,7 @@ void *proc_main(void *ptr)
117
118 if(!vdo_proc_net_dev) {
119 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_dev().");
130 - sunow = timems();
120 + sunow = time_usec();
121 vdo_proc_net_dev = do_proc_net_dev(rrd_update_every, (sutime_proc_net_dev > 0)?sunow - sutime_proc_net_dev:0ULL);
122 sutime_proc_net_dev = sunow;
123 }
@@ -135,7 +125,7 @@ void *proc_main(void *ptr)
125
126 if(!vdo_proc_diskstats) {
127 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_diskstats().");
138 - sunow = timems();
128 + sunow = time_usec();
129 vdo_proc_diskstats = do_proc_diskstats(rrd_update_every, (sutime_proc_diskstats > 0)?sunow - sutime_proc_diskstats:0ULL);
130 sutime_proc_diskstats = sunow;
131 }
@@ -143,7 +133,7 @@ void *proc_main(void *ptr)
133
134 if(!vdo_proc_net_snmp) {
135 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_snmp().");
146 - sunow = timems();
136 + sunow = time_usec();
137 vdo_proc_net_snmp = do_proc_net_snmp(rrd_update_every, (sutime_proc_net_snmp > 0)?sunow - sutime_proc_net_snmp:0ULL);
138 sutime_proc_net_snmp = sunow;
139 }
@@ -151,7 +141,7 @@ void *proc_main(void *ptr)
141
142 if(!vdo_proc_net_snmp6) {
143 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_snmp6().");
154 - sunow = timems();
144 + sunow = time_usec();
145 vdo_proc_net_snmp6 = do_proc_net_snmp6(rrd_update_every, (sutime_proc_net_snmp6 > 0)?sunow - sutime_proc_net_snmp6:0ULL);
146 sutime_proc_net_snmp6 = sunow;
147 }
@@ -159,7 +149,7 @@ void *proc_main(void *ptr)
149
150 if(!vdo_proc_net_netstat) {
151 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_netstat().");
162 - sunow = timems();
152 + sunow = time_usec();
153 vdo_proc_net_netstat = do_proc_net_netstat(rrd_update_every, (sutime_proc_net_netstat > 0)?sunow - sutime_proc_net_netstat:0ULL);
154 sutime_proc_net_netstat = sunow;
155 }
@@ -167,7 +157,7 @@ void *proc_main(void *ptr)
157
158 if(!vdo_proc_net_stat_conntrack) {
159 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_stat_conntrack().");
170 - sunow = timems();
160 + sunow = time_usec();
161 vdo_proc_net_stat_conntrack = do_proc_net_stat_conntrack(rrd_update_every, (sutime_proc_net_stat_conntrack > 0)?sunow - sutime_proc_net_stat_conntrack:0ULL);
162 sutime_proc_net_stat_conntrack = sunow;
163 }
@@ -175,7 +165,7 @@ void *proc_main(void *ptr)
165
166 if(!vdo_proc_net_ip_vs_stats) {
167 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_net_ip_vs_stats().");
178 - sunow = timems();
168 + sunow = time_usec();
169 vdo_proc_net_ip_vs_stats = do_proc_net_ip_vs_stats(rrd_update_every, (sutime_proc_net_ip_vs_stats > 0)?sunow - sutime_proc_net_ip_vs_stats:0ULL);
170 sutime_proc_net_ip_vs_stats = sunow;
171 }
@@ -183,7 +173,7 @@ void *proc_main(void *ptr)
173
174 if(!vdo_proc_net_stat_synproxy) {
175 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_net_stat_synproxy().");
186 - sunow = timems();
176 + sunow = time_usec();
177 vdo_proc_net_stat_synproxy = do_proc_net_stat_synproxy(rrd_update_every, (sutime_proc_net_stat_synproxy > 0)?sunow - sutime_proc_net_stat_synproxy:0ULL);
178 sutime_proc_net_stat_synproxy = sunow;
179 }
@@ -191,7 +181,7 @@ void *proc_main(void *ptr)
181
182 if(!vdo_proc_stat) {
183 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_stat().");
194 - sunow = timems();
184 + sunow = time_usec();
185 vdo_proc_stat = do_proc_stat(rrd_update_every, (sutime_proc_stat > 0)?sunow - sutime_proc_stat:0ULL);
186 sutime_proc_stat = sunow;
187 }
@@ -199,7 +189,7 @@ void *proc_main(void *ptr)
189
190 if(!vdo_proc_meminfo) {
191 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_meminfo().");
202 - sunow = timems();
192 + sunow = time_usec();
193 vdo_proc_meminfo = do_proc_meminfo(rrd_update_every, (sutime_proc_meminfo > 0)?sunow - sutime_proc_meminfo:0ULL);
194 sutime_proc_meminfo = sunow;
195 }
@@ -207,7 +197,7 @@ void *proc_main(void *ptr)
197
198 if(!vdo_proc_vmstat) {
199 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_vmstat().");
210 - sunow = timems();
200 + sunow = time_usec();
201 vdo_proc_vmstat = do_proc_vmstat(rrd_update_every, (sutime_proc_vmstat > 0)?sunow - sutime_proc_vmstat:0ULL);
202 sutime_proc_vmstat = sunow;
203 }
@@ -215,7 +205,7 @@ void *proc_main(void *ptr)
205
206 if(!vdo_proc_net_rpc_nfsd) {
207 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_rpc_nfsd().");
218 - sunow = timems();
208 + sunow = time_usec();
209 vdo_proc_net_rpc_nfsd = do_proc_net_rpc_nfsd(rrd_update_every, (sutime_proc_net_rpc_nfsd > 0)?sunow - sutime_proc_net_rpc_nfsd:0ULL);
210 sutime_proc_net_rpc_nfsd = sunow;
211 }
@@ -226,136 +216,7 @@ void *proc_main(void *ptr)
216 // --------------------------------------------------------------------
217
218 if(!vdo_cpu_netdata) {
229 - getrusage(RUSAGE_THREAD, &thread);
230 - getrusage(RUSAGE_SELF, &me);
231 -
232 - if(!stcpu_thread) stcpu_thread = rrdset_find("netdata.plugin_proc_cpu");
233 - if(!stcpu_thread) {
234 - stcpu_thread = rrdset_create("netdata", "plugin_proc_cpu", NULL, "proc.internal", NULL, "NetData Proc Plugin CPU usage", "milliseconds/s", 132000, rrd_update_every, RRDSET_TYPE_STACKED);
235 -
236 - rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL);
237 - rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL);
238 - }
239 - else rrdset_next(stcpu_thread);
240 -
241 - rrddim_set(stcpu_thread, "user" , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
242 - rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
243 - rrdset_done(stcpu_thread);
244 -
245 - // ----------------------------------------------------------------
246 -
247 - if(!stcpu) stcpu = rrdset_find("netdata.server_cpu");
248 - if(!stcpu) {
249 - stcpu = rrdset_create("netdata", "server_cpu", NULL, "netdata", NULL, "NetData CPU usage", "milliseconds/s", 130000, rrd_update_every, RRDSET_TYPE_STACKED);
250 -
251 - rrddim_add(stcpu, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL);
252 - rrddim_add(stcpu, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL);
253 - }
254 - else rrdset_next(stcpu);
255 -
256 - rrddim_set(stcpu, "user" , me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec);
257 - rrddim_set(stcpu, "system", me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec);
258 - rrdset_done(stcpu);
259 -
260 - // ----------------------------------------------------------------
261 -
262 - if(!stclients) stclients = rrdset_find("netdata.clients");
263 - if(!stclients) {
264 - stclients = rrdset_create("netdata", "clients", NULL, "netdata", NULL, "NetData Web Clients", "connected clients", 130100, rrd_update_every, RRDSET_TYPE_LINE);
265 -
266 - rrddim_add(stclients, "clients", NULL, 1, 1, RRDDIM_ABSOLUTE);
267 - }
268 - else rrdset_next(stclients);
269 -
270 - rrddim_set(stclients, "clients", global_statistics.connected_clients);
271 - rrdset_done(stclients);
272 -
273 - // ----------------------------------------------------------------
274 -
275 - if(!streqs) streqs = rrdset_find("netdata.requests");
276 - if(!streqs) {
277 - streqs = rrdset_create("netdata", "requests", NULL, "netdata", NULL, "NetData Web Requests", "requests/s", 130200, rrd_update_every, RRDSET_TYPE_LINE);
278 -
279 - rrddim_add(streqs, "requests", NULL, 1, 1, RRDDIM_INCREMENTAL);
280 - }
281 - else rrdset_next(streqs);
282 -
283 - rrddim_set(streqs, "requests", global_statistics.web_requests);
284 - rrdset_done(streqs);
285 -
286 - // ----------------------------------------------------------------
287 -
288 - if(!stbytes) stbytes = rrdset_find("netdata.net");
289 - if(!stbytes) {
290 - stbytes = rrdset_create("netdata", "net", NULL, "netdata", NULL, "NetData Network Traffic", "kilobits/s", 130300, rrd_update_every, RRDSET_TYPE_AREA);
291 -
292 - rrddim_add(stbytes, "in", NULL, 8, 1024, RRDDIM_INCREMENTAL);
293 - rrddim_add(stbytes, "out", NULL, -8, 1024, RRDDIM_INCREMENTAL);
294 - }
295 - else rrdset_next(stbytes);
296 -
297 - rrddim_set(stbytes, "in", global_statistics.bytes_received);
298 - rrddim_set(stbytes, "out", global_statistics.bytes_sent);
299 - rrdset_done(stbytes);
300 -
301 - // ----------------------------------------------------------------
302 -
303 - if(!stduration) stduration = rrdset_find("netdata.response_time");
304 - if(!stduration) {
305 - stduration = rrdset_create("netdata", "response_time", NULL, "netdata", NULL, "NetData Average API Response Time", "ms/request", 130400, rrd_update_every, RRDSET_TYPE_LINE);
306 -
307 - rrddim_add(stduration, "response_time", "response time", 1, 1000, RRDDIM_ABSOLUTE);
308 - }
309 - else rrdset_next(stduration);
310 -
311 - unsigned long long gweb_usec = global_statistics.web_usec;
312 - unsigned long long gweb_requests = global_statistics.web_requests;
313 -
314 - unsigned long long web_usec = gweb_usec - old_web_usec;
315 - unsigned long long web_requests = gweb_requests - old_web_requests;
316 -
317 - old_web_usec = gweb_usec;
318 - old_web_requests = gweb_requests;
319 -
320 - if(web_requests)
321 - average_response_time = web_usec / web_requests;
322 -
323 - if(average_response_time != -1)
324 - rrddim_set(stduration, "response_time", average_response_time);
325 -
326 - rrdset_done(stduration);
327 -
328 - // ----------------------------------------------------------------
329 -
330 - if(!stcompression) stcompression = rrdset_find("netdata.compression_ratio");
331 - if(!stcompression) {
332 - stcompression = rrdset_create("netdata", "compression_ratio", NULL, "netdata", NULL, "NetData API Responses Compression Savings Ratio", "percentage", 130500, rrd_update_every, RRDSET_TYPE_LINE);
333 -
334 - rrddim_add(stcompression, "savings", NULL, 1, 1000, RRDDIM_ABSOLUTE);
335 - }
336 - else rrdset_next(stcompression);
337 -
338 - // since we don't lock here to read the global statistics
339 - // read the smaller value first
340 - unsigned long long gcompressed_content_size = global_statistics.compressed_content_size;
341 - unsigned long long gcontent_size = global_statistics.content_size;
342 -
343 - unsigned long long compressed_content_size = gcompressed_content_size - old_compressed_content_size;
344 - unsigned long long content_size = gcontent_size - old_content_size;
345 -
346 - old_compressed_content_size = gcompressed_content_size;
347 - old_content_size = gcontent_size;
348 -
349 - if(content_size && content_size >= compressed_content_size)
350 - compression_ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size;
351 -
352 - if(compression_ratio != -1)
353 - rrddim_set(stcompression, "savings", compression_ratio);
354 -
355 - rrdset_done(stcompression);
356 -
357 - // ----------------------------------------------------------------
358 -
219 + global_statistics_charts();
220 registry_statistics();
221 }
222 }
src/rrd.c
+3 -3
@@ -600,7 +600,7 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier
600 error("File %s does not have the same refresh frequency. Clearing it.", fullfilename);
601 bzero(rd, size);
602 }
603 - else if(usecdiff(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) {
603 + else if(usec_dt(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) {
604 errno = 0;
605 error("File %s is too old. Clearing it.", fullfilename);
606 bzero(rd, size);
@@ -922,7 +922,7 @@ void rrdset_next(RRDSET *st)
922 if(likely(st->last_collected_time.tv_sec)) {
923 struct timeval now;
924 gettimeofday(&now, NULL);
925 - microseconds = usecdiff(&now, &st->last_collected_time);
925 + microseconds = usec_dt(&now, &st->last_collected_time);
926 }
927 // prevent infinite loop
928 else microseconds = st->update_every * 1000000ULL;
@@ -1001,7 +1001,7 @@ unsigned long long rrdset_done(RRDSET *st)
1001 }
1002
1003 // check if we will re-write the entire data set
1004 - if(unlikely(usecdiff(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL)) {
1004 + if(unlikely(usec_dt(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL)) {
1005 info("%s: too old data (last updated at %ld.%ld, last collected at %ld.%ld). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec);
1006 rrdset_reset(st);
1007
src/sys_fs_cgroup.c
+4 -4
@@ -1287,11 +1287,11 @@ void *cgroups_main(void *ptr)
1287 if(unlikely(netdata_exit)) break;
1288
1289 // delay until it is our time to run
1290 - while((sunow = timems()) < sunext)
1291 - usecsleep(sunext - sunow);
1290 + while((sunow = time_usec()) < sunext)
1291 + sleep_usec(sunext - sunow);
1292
1293 // find the next time we need to run
1294 - while(timems() > sunext)
1294 + while(time_usec() > sunext)
1295 sunext += rrd_update_every * 1000000ULL;
1296
1297 if(unlikely(netdata_exit)) break;
@@ -1300,7 +1300,7 @@ void *cgroups_main(void *ptr)
1300
1301 if(!vdo_sys_fs_cgroup) {
1302 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_sys_fs_cgroup().");
1303 - sunow = timems();
1303 + sunow = time_usec();
1304 vdo_sys_fs_cgroup = do_sys_fs_cgroup(rrd_update_every, (sutime_sys_fs_cgroup > 0)?sunow - sutime_sys_fs_cgroup:0ULL);
1305 sutime_sys_fs_cgroup = sunow;
1306 }
src/web_client.c
+15 -21
@@ -113,7 +113,7 @@ struct web_client *web_client_create(int listener)
113 w->next = web_clients;
114 web_clients = w;
115
116 - global_statistics.connected_clients++;
116 + web_client_connected();
117
118 return(w);
119 }
@@ -136,18 +136,11 @@ void web_client_reset(struct web_client *w) {
136 // --------------------------------------------------------------------
137 // global statistics
138
139 - if(web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
140 - global_statistics_lock();
141 -
142 - global_statistics.web_requests++;
143 - global_statistics.web_usec += usecdiff(&tv, &w->tv_in);
144 - global_statistics.bytes_received += w->stats_received_bytes;
145 - global_statistics.bytes_sent += w->stats_sent_bytes;
146 - global_statistics.content_size += size;
147 - global_statistics.compressed_content_size += sent;
148 -
149 - if(web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
150 - global_statistics_unlock();
139 + finished_web_request_statistics(usec_dt(&tv, &w->tv_in),
140 + w->stats_received_bytes,
141 + w->stats_sent_bytes,
142 + size,
143 + sent);
144
145 w->stats_received_bytes = 0;
146 w->stats_sent_bytes = 0;
@@ -159,9 +152,9 @@ void web_client_reset(struct web_client *w) {
152 log_access("%llu: (sent/all = %zu/%zu bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %s: %d '%s'",
153 w->id,
154 sent, size, -((size > 0) ? ((size - sent) / (double) size * 100.0) : 0.0),
162 - usecdiff(&w->tv_ready, &w->tv_in) / 1000.0,
163 - usecdiff(&tv, &w->tv_ready) / 1000.0,
164 - usecdiff(&tv, &w->tv_in) / 1000.0,
155 + usec_dt(&w->tv_ready, &w->tv_in) / 1000.0,
156 + usec_dt(&tv, &w->tv_ready) / 1000.0,
157 + usec_dt(&tv, &w->tv_in) / 1000.0,
158 (w->mode == WEB_CLIENT_MODE_FILECOPY) ? "filecopy" : ((w->mode == WEB_CLIENT_MODE_OPTIONS)
159 ? "options" : "data"),
160 w->response.code,
@@ -236,7 +229,7 @@ struct web_client *web_client_free(struct web_client *w) {
229 if(w->ofd != -1 && w->ofd != w->ifd) close(w->ofd);
230 freez(w);
231
239 - global_statistics.connected_clients--;
232 + web_client_disconnected();
233
234 return(n);
235 }
@@ -1668,7 +1661,10 @@ void web_client_process(struct web_client *w) {
1661 static uint32_t hash_api = 0, hash_netdata_conf = 0, hash_data = 0, hash_datasource = 0, hash_graph = 0,
1662 hash_list = 0, hash_all_json = 0, hash_exit = 0, hash_debug = 0, hash_mirror = 0;
1663
1671 - if(unlikely(!hash_api)) {
1664 + // start timing us
1665 + gettimeofday(&w->tv_in, NULL);
1666 +
1667 + if(unlikely(!hash_api)) {
1668 hash_api = simple_hash("api");
1669 hash_netdata_conf = simple_hash("netdata.conf");
1670 hash_data = simple_hash(WEB_PATH_DATA);
@@ -1703,7 +1699,7 @@ void web_client_process(struct web_client *w) {
1699 }
1700 }
1701 else if(what_to_do > 0) {
1706 - strcpy(w->last_url, "not a valid request");
1702 + // strcpy(w->last_url, "not a valid request");
1703
1704 debug(D_WEB_CLIENT_ACCESS, "%llu: Cannot understand '%s'.", w->id, w->response.data->buffer);
1705
@@ -1712,8 +1708,6 @@ void web_client_process(struct web_client *w) {
1708 buffer_strcat(w->response.data, "I don't understand you...\r\n");
1709 }
1710 else { // what_to_do == 0
1715 - gettimeofday(&w->tv_in, NULL);
1716 -
1711 if(w->mode == WEB_CLIENT_MODE_OPTIONS) {
1712 code = 200;
1713 w->response.data->contenttype = CT_TEXT_PLAIN;
web/demosites.html
+8 -8
@@ -945,8 +945,8 @@ body {
945 data-width="100%"
946 data-height="100%"
947 data-after="-300"
948 - data-colors="#558855"
949 - data-show-value-of-response_time-at="netdata.response_time1"
948 + data-colors="#558855 #356835"
949 + data-show-value-of-average-at="netdata.response_time1"
950 ></div>
951 </div>
952
@@ -964,8 +964,8 @@ body {
964 data-width="100%"
965 data-height="100%"
966 data-after="-300"
967 - data-colors="#885555"
968 - data-show-value-of-response_time-at="netdata.response_time2"
967 + data-colors="#885555 #683535"
968 + data-show-value-of-average-at="netdata.response_time2"
969 ></div>
970 </div>
971
@@ -983,8 +983,8 @@ body {
983 data-width="100%"
984 data-height="100%"
985 data-after="-300"
986 - data-colors="#555588"
987 - data-show-value-of-response_time-at="netdata.response_time3"
986 + data-colors="#555588 #353568"
987 + data-show-value-of-average-at="netdata.response_time3"
988 ></div>
989 </div>
990
@@ -1002,8 +1002,8 @@ body {
1002 data-width="100%"
1003 data-height="100%"
1004 data-after="-300"
1005 - data-colors="#885588"
1006 - data-show-value-of-response_time-at="netdata.response_time4"
1005 + data-colors="#885588 #683568"
1006 + data-show-value-of-average-at="netdata.response_time4"
1007 ></div>
1008 </div>
1009