Move cpufreq python module to proc plugin (#4562)
* Add time_in_state file to processing * Add time_in_state calculation logic and TODOs * Add time_in_state calculation and switching * Final cleanup * Add info messages * Remove excessive array * Rename chart and disable python module
Vladimir Kobal committed
Nov 15, 2018 at 18:43 UTC
9b56c2474d847120e113aa9d7cd526e541c98de4
2 files changed
+179
-27
collectors/proc.plugin/proc_stat.c
+178
-26
@@ -12,9 +12,23 @@ struct per_core_single_number_file {
12
RRDDIM *rd;
13
};
14
15
+struct last_ticks {
16
+ collected_number frequency;
17
+ collected_number ticks;
18
+};
19
+
20
+// This is an extension of struct per_core_single_number_file at CPU_FREQ_INDEX.
21
+// Either scaling_cur_freq or time_in_state file is used at one time.
22
+struct per_core_time_in_state_file {
23
+ const char *filename;
24
+ procfile *ff;
25
+ size_t last_ticks_len;
26
+ struct last_ticks *last_ticks;
27
+};
28
+
29
#define CORE_THROTTLE_COUNT_INDEX 0
30
#define PACKAGE_THROTTLE_COUNT_INDEX 1
17
-#define SCALING_CUR_FREQ_INDEX 2
31
+#define CPU_FREQ_INDEX 2
32
#define PER_CORE_FILES 3
33
34
struct cpu_chart {
@@ -33,6 +47,8 @@ struct cpu_chart {
47
RRDDIM *rd_guest_nice;
48
49
struct per_core_single_number_file files[PER_CORE_FILES];
50
+
51
+ struct per_core_time_in_state_file time_in_state_files;
52
};
53
54
static int keep_per_core_fds_open = CONFIG_BOOLEAN_YES;
@@ -87,7 +103,6 @@ static int read_per_core_files(struct cpu_chart *all_cpu_charts, size_t len, siz
103
f->found = 1;
104
105
f->value = str2ll(buf, NULL);
90
- // info("read '%s', parsed as " COLLECTED_NUMBER_FORMAT, buf, f->value);
106
if(likely(f->value != 0))
107
files_nonzero++;
108
}
@@ -101,6 +116,112 @@ static int read_per_core_files(struct cpu_chart *all_cpu_charts, size_t len, siz
116
return (int)files_nonzero;
117
}
118
119
+static int read_per_core_time_in_state_files(struct cpu_chart *all_cpu_charts, size_t len, size_t index) {
120
+ size_t x, files_read = 0, files_nonzero = 0;
121
+
122
+ for(x = 0; x < len ; x++) {
123
+ struct per_core_single_number_file *f = &all_cpu_charts[x].files[index];
124
+ struct per_core_time_in_state_file *tsf = &all_cpu_charts[x].time_in_state_files;
125
+
126
+ f->found = 0;
127
+
128
+ if(unlikely(!tsf->filename))
129
+ continue;
130
+
131
+ if(unlikely(!tsf->ff)) {
132
+ tsf->ff = procfile_open(tsf->filename, " \t:", PROCFILE_FLAG_DEFAULT);
133
+ if(unlikely(!tsf->ff))
134
+ {
135
+ error("Cannot open file '%s'", tsf->filename);
136
+ continue;
137
+ }
138
+ }
139
+
140
+ tsf->ff = procfile_readall(tsf->ff);
141
+ if(unlikely(!tsf->ff)) {
142
+ error("Cannot read file '%s'", tsf->filename);
143
+ procfile_close(tsf->ff);
144
+ tsf->ff = NULL;
145
+ continue;
146
+ }
147
+ else {
148
+ // successful read
149
+
150
+ size_t lines = procfile_lines(tsf->ff), l;
151
+ size_t words;
152
+ unsigned long long total_ticks_since_last = 0, avg_freq = 0;
153
+
154
+ // Check if there is at least one frequency in time_in_state
155
+ if (procfile_word(tsf->ff, 0)[0] == '\0') {
156
+ if(unlikely(keep_per_core_fds_open != CONFIG_BOOLEAN_YES)) {
157
+ procfile_close(tsf->ff);
158
+ tsf->ff = NULL;
159
+ }
160
+ // TODO: Is there a better way to avoid spikes than calculating the average over
161
+ // the whole period under schedutil governor?
162
+ // freez(tsf->last_ticks);
163
+ // tsf->last_ticks = NULL;
164
+ // tsf->last_ticks_len = 0;
165
+ continue;
166
+ }
167
+
168
+ if (unlikely(tsf->last_ticks_len < lines || tsf->last_ticks == NULL)) {
169
+ tsf->last_ticks = reallocz(tsf->last_ticks, sizeof(struct last_ticks) * lines);
170
+ memset(tsf->last_ticks, 0, sizeof(struct last_ticks) * lines);
171
+ tsf->last_ticks_len = lines;
172
+ }
173
+
174
+ f->value = 0;
175
+
176
+ for(l = 0; l < lines - 1 ;l++) {
177
+ unsigned long long frequency = 0, ticks = 0, ticks_since_last = 0;
178
+
179
+ words = procfile_linewords(tsf->ff, l);
180
+ if(unlikely(words < 2)) {
181
+ error("Cannot read time_in_state line. Expected 2 params, read %zu.", words);
182
+ continue;
183
+ }
184
+ frequency = str2ull(procfile_lineword(tsf->ff, l, 0));
185
+ ticks = str2ull(procfile_lineword(tsf->ff, l, 1));
186
+
187
+ // It is assumed that frequencies are static and sorted
188
+ ticks_since_last = ticks - tsf->last_ticks[l].ticks;
189
+ tsf->last_ticks[l].frequency = frequency;
190
+ tsf->last_ticks[l].ticks = ticks;
191
+
192
+ total_ticks_since_last += ticks_since_last;
193
+ avg_freq += frequency * ticks_since_last;
194
+
195
+ }
196
+
197
+ if (likely(total_ticks_since_last)) {
198
+ avg_freq /= total_ticks_since_last;
199
+ f->value = avg_freq;
200
+ }
201
+
202
+ if(unlikely(keep_per_core_fds_open != CONFIG_BOOLEAN_YES)) {
203
+ procfile_close(tsf->ff);
204
+ tsf->ff = NULL;
205
+ }
206
+ }
207
+
208
+ files_read++;
209
+
210
+ f->found = 1;
211
+
212
+ if(likely(f->value != 0))
213
+ files_nonzero++;
214
+ }
215
+
216
+ if(unlikely(files_read == 0))
217
+ return -1;
218
+
219
+ if(unlikely(files_nonzero == 0))
220
+ return 0;
221
+
222
+ return (int)files_nonzero;
223
+}
224
+
225
static void chart_per_core_files(struct cpu_chart *all_cpu_charts, size_t len, size_t index, RRDSET *st, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm) {
226
size_t x;
227
for(x = 0; x < len ; x++) {
@@ -122,10 +243,11 @@ int do_proc_stat(int update_every, usec_t dt) {
243
static struct cpu_chart *all_cpu_charts = NULL;
244
static size_t all_cpu_charts_size = 0;
245
static procfile *ff = NULL;
125
- static int do_cpu = -1, do_cpu_cores = -1, do_interrupts = -1, do_context = -1, do_forks = -1, do_processes = -1, do_core_throttle_count = -1, do_package_throttle_count = -1, do_scaling_cur_freq = -1;
246
+ static int do_cpu = -1, do_cpu_cores = -1, do_interrupts = -1, do_context = -1, do_forks = -1, do_processes = -1, do_core_throttle_count = -1, do_package_throttle_count = -1, do_cpu_freq = -1;
247
static uint32_t hash_intr, hash_ctxt, hash_processes, hash_procs_running, hash_procs_blocked;
127
- static char *core_throttle_count_filename = NULL, *package_throttle_count_filename = NULL, *scaling_cur_freq_filename = NULL;
248
+ static char *core_throttle_count_filename = NULL, *package_throttle_count_filename = NULL, *scaling_cur_freq_filename = NULL, *time_in_state_filename = NULL;
249
static RRDVAR *cpus_var = NULL;
250
+ static int accurate_freq_avail = 0, accurate_freq_is_used = 0;
251
size_t cores_found = (size_t)processors;
252
253
if(unlikely(do_cpu == -1)) {
@@ -137,25 +259,25 @@ int do_proc_stat(int update_every, usec_t dt) {
259
do_processes = config_get_boolean("plugin:proc:/proc/stat", "processes running", CONFIG_BOOLEAN_YES);
260
261
// give sane defaults based on the number of processors
140
- if(processors > 50) {
262
+ if(unlikely(processors > 50)) {
263
// the system has too many processors
264
keep_per_core_fds_open = CONFIG_BOOLEAN_NO;
265
do_core_throttle_count = CONFIG_BOOLEAN_NO;
266
do_package_throttle_count = CONFIG_BOOLEAN_NO;
145
- do_scaling_cur_freq = CONFIG_BOOLEAN_NO;
267
+ do_cpu_freq = CONFIG_BOOLEAN_NO;
268
}
269
else {
270
// the system has a reasonable number of processors
271
keep_per_core_fds_open = CONFIG_BOOLEAN_YES;
272
do_core_throttle_count = CONFIG_BOOLEAN_AUTO;
273
do_package_throttle_count = CONFIG_BOOLEAN_NO;
152
- do_scaling_cur_freq = CONFIG_BOOLEAN_NO;
274
+ do_cpu_freq = CONFIG_BOOLEAN_YES;
275
}
276
277
keep_per_core_fds_open = config_get_boolean("plugin:proc:/proc/stat", "keep per core files open", keep_per_core_fds_open);
278
do_core_throttle_count = config_get_boolean_ondemand("plugin:proc:/proc/stat", "core_throttle_count", do_core_throttle_count);
279
do_package_throttle_count = config_get_boolean_ondemand("plugin:proc:/proc/stat", "package_throttle_count", do_package_throttle_count);
158
- do_scaling_cur_freq = config_get_boolean_ondemand("plugin:proc:/proc/stat", "scaling_cur_freq", do_scaling_cur_freq);
280
+ do_cpu_freq = config_get_boolean_ondemand("plugin:proc:/proc/stat", "cpu frequency", do_cpu_freq);
281
282
hash_intr = simple_hash("intr");
283
hash_ctxt = simple_hash("ctxt");
@@ -172,6 +294,9 @@ int do_proc_stat(int update_every, usec_t dt) {
294
295
snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/cpu/%s/cpufreq/scaling_cur_freq");
296
scaling_cur_freq_filename = config_get("plugin:proc:/proc/stat", "scaling_cur_freq filename to monitor", filename);
297
+
298
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/cpu/%s/cpufreq/stats/time_in_state");
299
+ time_in_state_filename = config_get("plugin:proc:/proc/stat", "time_in_state filename to monitor", filename);
300
}
301
302
if(unlikely(!ff)) {
@@ -202,7 +327,7 @@ int do_proc_stat(int update_every, usec_t dt) {
327
}
328
329
size_t core = (row_key[3] == '\0') ? 0 : str2ul(&row_key[3]) + 1;
205
- if(core > 0) cores_found = core;
330
+ if(likely(core > 0)) cores_found = core;
331
332
if(likely((core == 0 && do_cpu) || (core > 0 && do_cpu_cores))) {
333
char *id;
@@ -227,7 +352,7 @@ int do_proc_stat(int update_every, usec_t dt) {
352
char *title, *type, *context, *family;
353
long priority;
354
230
- if(core >= all_cpu_charts_size) {
355
+ if(unlikely(core >= all_cpu_charts_size)) {
356
size_t old_cpu_charts_size = all_cpu_charts_size;
357
all_cpu_charts_size = core + 1;
358
all_cpu_charts = reallocz(all_cpu_charts, sizeof(struct cpu_chart) * all_cpu_charts_size);
@@ -238,7 +363,7 @@ int do_proc_stat(int update_every, usec_t dt) {
363
if(unlikely(!cpu_chart->st)) {
364
cpu_chart->id = strdupz(id);
365
241
- if(core == 0) {
366
+ if(unlikely(core == 0)) {
367
title = "Total CPU utilization";
368
type = "system";
369
context = "system.cpu";
@@ -252,9 +377,6 @@ int do_proc_stat(int update_every, usec_t dt) {
377
family = "utilization";
378
priority = NETDATA_CHART_PRIO_CPU_PER_CORE;
379
255
- // TODO: check for /sys/devices/system/cpu/cpu*/cpufreq/scaling_cur_freq
256
- // TODO: check for /sys/devices/system/cpu/cpu*/cpufreq/stats/time_in_state
257
-
380
char filename[FILENAME_MAX + 1];
381
struct stat stbuf;
382
@@ -276,12 +398,23 @@ int do_proc_stat(int update_every, usec_t dt) {
398
}
399
}
400
279
- if(do_scaling_cur_freq != CONFIG_BOOLEAN_NO) {
401
+ if(do_cpu_freq != CONFIG_BOOLEAN_NO) {
402
+
403
snprintfz(filename, FILENAME_MAX, scaling_cur_freq_filename, id);
404
+
405
+ if (stat(filename, &stbuf) == 0) {
406
+ cpu_chart->files[CPU_FREQ_INDEX].filename = strdupz(filename);
407
+ cpu_chart->files[CPU_FREQ_INDEX].fd = -1;
408
+ do_cpu_freq = CONFIG_BOOLEAN_YES;
409
+ }
410
+
411
+ snprintfz(filename, FILENAME_MAX, time_in_state_filename, id);
412
+
413
if (stat(filename, &stbuf) == 0) {
282
- cpu_chart->files[SCALING_CUR_FREQ_INDEX].filename = strdupz(filename);
283
- cpu_chart->files[SCALING_CUR_FREQ_INDEX].fd = -1;
284
- do_scaling_cur_freq = CONFIG_BOOLEAN_YES;
414
+ cpu_chart->time_in_state_files.filename = strdupz(filename);
415
+ cpu_chart->time_in_state_files.ff = NULL;
416
+ do_cpu_freq = CONFIG_BOOLEAN_YES;
417
+ accurate_freq_avail = 1;
418
}
419
}
420
}
@@ -532,21 +665,40 @@ int do_proc_stat(int update_every, usec_t dt) {
665
}
666
}
667
535
- if(likely(do_scaling_cur_freq != CONFIG_BOOLEAN_NO)) {
536
- int r = read_per_core_files(&all_cpu_charts[1], all_cpu_charts_size - 1, SCALING_CUR_FREQ_INDEX);
537
- if(likely(r != -1 && (do_scaling_cur_freq == CONFIG_BOOLEAN_YES || r > 0))) {
538
- do_scaling_cur_freq = CONFIG_BOOLEAN_YES;
668
+ if(likely(do_cpu_freq != CONFIG_BOOLEAN_NO)) {
669
+ char filename[FILENAME_MAX + 1];
670
+ int r = 0;
671
+
672
+ if (accurate_freq_avail) {
673
+ r = read_per_core_time_in_state_files(&all_cpu_charts[1], all_cpu_charts_size - 1, CPU_FREQ_INDEX);
674
+ if(r > 0 && !accurate_freq_is_used) {
675
+ accurate_freq_is_used = 1;
676
+ snprintfz(filename, FILENAME_MAX, time_in_state_filename, "cpu*");
677
+ info("cpufreq is using %s", filename);
678
+ }
679
+ }
680
+ if (r < 1) {
681
+ r = read_per_core_files(&all_cpu_charts[1], all_cpu_charts_size - 1, CPU_FREQ_INDEX);
682
+ if(accurate_freq_is_used) {
683
+ accurate_freq_is_used = 0;
684
+ snprintfz(filename, FILENAME_MAX, scaling_cur_freq_filename, "cpu*");
685
+ info("cpufreq fell back to %s", filename);
686
+ }
687
+ }
688
+
689
+ if(likely(r != -1 && (do_cpu_freq == CONFIG_BOOLEAN_YES || r > 0))) {
690
+ do_cpu_freq = CONFIG_BOOLEAN_YES;
691
692
static RRDSET *st_scaling_cur_freq = NULL;
693
694
if(unlikely(!st_scaling_cur_freq))
695
st_scaling_cur_freq = rrdset_create_localhost(
696
"cpu"
545
- , "scaling_cur_freq"
697
+ , "cpufreq"
698
, NULL
699
, "cpufreq"
548
- , "cpu.scaling_cur_freq"
549
- , "Per CPU Core, Current CPU Scaling Frequency"
700
+ , "cpufreq.cpufreq"
701
+ , "Current CPU Frequency"
702
, "MHz"
703
, PLUGIN_PROC_NAME
704
, PLUGIN_PROC_MODULE_STAT_NAME
@@ -557,7 +709,7 @@ int do_proc_stat(int update_every, usec_t dt) {
709
else
710
rrdset_next(st_scaling_cur_freq);
711
560
- chart_per_core_files(&all_cpu_charts[1], all_cpu_charts_size - 1, SCALING_CUR_FREQ_INDEX, st_scaling_cur_freq, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
712
+ chart_per_core_files(&all_cpu_charts[1], all_cpu_charts_size - 1, CPU_FREQ_INDEX, st_scaling_cur_freq, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
713
rrdset_done(st_scaling_cur_freq);
714
}
715
}
collectors/python.d.plugin/python.d.plugin.in
+1
-1
@@ -56,7 +56,7 @@ BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
56
57
58
MODULE_EXTENSION = '.chart.py'
59
-OBSOLETE_MODULES = ['apache_cache', 'gunicorn_log', 'nginx_log']
59
+OBSOLETE_MODULES = ['apache_cache', 'gunicorn_log', 'nginx_log', 'cpufreq']
60
61
62
def module_ok(m):