@cryptotaxi247 / netdata-1 / commits / 0b7ba99b5

[collector/proc.plugin] Add /proc/pagetypeinfo parser (#6843)

* [proc.plugin/proc_pagetypeinfo] Initial commit * [Fix] Generate graphs for pagetypeinfo * [Fix] Create node/zone/type graphs * [Fix] Use directly size and order * [Add] Configuration handling * [Imp] Changed SetId to identify NodeNumber * [Fix] Standard name for chart priority and value * [Fix] use dynamic pagesize * [Enh] allow prefix for containerized netdata * [Fix] global system graph always on, but for explicit no * [Fix] Add more checks for pageorders_cnt and really use it * [Enh] Special config value of netdata_zero_metrics_enabled * [Fix] Check we parsed at least a valid line

Adrien Mahieux committed Oct 21, 2019 at 09:51 UTC 0b7ba99b5e344620ea3f1e633ba574590b9eda82
6 files changed +338
CMakeLists.txt
+1
@@ -415,6 +415,7 @@ set(PROC_PLUGIN_FILES
415 collectors/proc.plugin/proc_softirqs.c
416 collectors/proc.plugin/proc_loadavg.c
417 collectors/proc.plugin/proc_meminfo.c
418 + collectors/proc.plugin/proc_pagetypeinfo.c
419 collectors/proc.plugin/proc_net_dev.c
420 collectors/proc.plugin/proc_net_ip_vs_stats.c
421 collectors/proc.plugin/proc_net_netstat.c
Makefile.am
+1
@@ -262,6 +262,7 @@ PROC_PLUGIN_FILES = \
262 collectors/proc.plugin/proc_softirqs.c \
263 collectors/proc.plugin/proc_loadavg.c \
264 collectors/proc.plugin/proc_meminfo.c \
265 + collectors/proc.plugin/proc_pagetypeinfo.c \
266 collectors/proc.plugin/proc_net_dev.c \
267 collectors/proc.plugin/proc_net_ip_vs_stats.c \
268 collectors/proc.plugin/proc_net_netstat.c \
collectors/all.h
+1
@@ -90,6 +90,7 @@
90 #define NETDATA_CHART_PRIO_MEM_KSM_RATIOS 1302
91 #define NETDATA_CHART_PRIO_MEM_NUMA 1400
92 #define NETDATA_CHART_PRIO_MEM_NUMA_NODES 1410
93 +#define NETDATA_CHART_PRIO_MEM_PAGEFRAG 1450
94 #define NETDATA_CHART_PRIO_MEM_HW 1500
95 #define NETDATA_CHART_PRIO_MEM_HW_ECC_CE 1550
96 #define NETDATA_CHART_PRIO_MEM_HW_ECC_UE 1560
collectors/proc.plugin/plugin_proc.c
+1
@@ -32,6 +32,7 @@ static struct proc_module {
32 { .name = "/sys/block/zram", .dim = "zram", .func = do_sys_block_zram },
33 { .name = "/sys/devices/system/edac/mc", .dim = "ecc", .func = do_proc_sys_devices_system_edac_mc },
34 { .name = "/sys/devices/system/node", .dim = "numa", .func = do_proc_sys_devices_system_node },
35 + { .name = "/proc/pagetypeinfo", .dim = "pagetypeinfo", .func = do_proc_pagetypeinfo },
36
37 // network metrics
38 { .name = "/proc/net/dev", .dim = "netdev", .func = do_proc_net_dev },
collectors/proc.plugin/plugin_proc.h
+1
@@ -55,6 +55,7 @@ extern int do_proc_net_sockstat6(int update_every, usec_t dt);
55 extern int do_proc_net_sctp_snmp(int update_every, usec_t dt);
56 extern int do_ipc(int update_every, usec_t dt);
57 extern int do_sys_class_power_supply(int update_every, usec_t dt);
58 +extern int do_proc_pagetypeinfo(int update_every, usec_t dt);
59 extern int get_numa_node_count(void);
60
61 // metrics that need to be shared among data collectors
collectors/proc.plugin/proc_pagetypeinfo.c new
+333
@@ -0,0 +1,333 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "plugin_proc.h"
4 +
5 +// For ULONG_MAX
6 +#include <limits.h>
7 +
8 +#define PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME "/proc/pagetypeinfo"
9 +#define CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME
10 +
11 +// Zone struct is pglist_data, in include/linux/mmzone.h
12 +// MAX_NR_ZONES is from __MAX_NR_ZONE, which is the last value of the enum.
13 +#define MAX_PAGETYPE_ORDER 11
14 +
15 +// Names are in mm/page_alloc.c :: migratetype_names. Max size = 10.
16 +#define MAX_ZONETYPE_NAME 16
17 +#define MAX_PAGETYPE_NAME 16
18 +
19 +// Defined in include/linux/mmzone.h as __MAX_NR_ZONE (last enum of zone_type)
20 +#define MAX_ZONETYPE 6
21 +// Defined in include/linux/mmzone.h as MIGRATE_TYPES (last enum of migratetype)
22 +#define MAX_PAGETYPE 7
23 +
24 +
25 +//
26 +// /proc/pagetypeinfo is declared in mm/vmstat.c :: init_mm_internals
27 +//
28 +
29 +// One line of /proc/pagetypeinfo
30 +struct pageline {
31 + int node;
32 + char *zone;
33 + char *type;
34 + int line;
35 + uint64_t free_pages_size[MAX_PAGETYPE_ORDER];
36 + RRDDIM *rd[MAX_PAGETYPE_ORDER];
37 +};
38 +
39 +// Sum of all orders
40 +struct systemorder {
41 + uint64_t size;
42 + RRDDIM *rd;
43 +};
44 +
45 +
46 +static inline uint64_t pageline_total_count(struct pageline *p) {
47 + uint64_t sum = 0, o;
48 + for (o=0; o<MAX_PAGETYPE_ORDER; o++)
49 + sum += p->free_pages_size[o];
50 + return sum;
51 +}
52 +
53 +// Check if a line of /proc/pagetypeinfo is valid to use
54 +// Free block lines starts by "Node" && 4th col is "type"
55 +#define pagetypeinfo_line_valid(ff, l) (strncmp(procfile_lineword(ff, l, 0), "Node", 4) == 0 && strncmp(procfile_lineword(ff, l, 4), "type", 4) == 0)
56 +
57 +// Dimension name from the order
58 +#define dim_name(s, o, pagesize) (snprintfz(s, 16,"%ldKB (%lu)", (1 << o) * pagesize / 1024, o))
59 +
60 +int do_proc_pagetypeinfo(int update_every, usec_t dt) {
61 + (void)dt;
62 +
63 + // Config
64 + static int do_global, do_detail;
65 + static SIMPLE_PATTERN *filter_types = NULL;
66 +
67 + // Counters from parsing the file, that doesn't change after boot
68 + static struct systemorder systemorders[MAX_PAGETYPE_ORDER] = {};
69 + static struct pageline* pagelines = NULL;
70 + static long pagesize = 0;
71 + static size_t pageorders_cnt = 0, pagelines_cnt = 0, ff_lines = 0;
72 +
73 + // Handle
74 + static procfile *ff = NULL;
75 + static char ff_path[FILENAME_MAX + 1];
76 +
77 + // RRD Sets
78 + static RRDSET *st_order = NULL;
79 + static RRDSET **st_nodezonetype = NULL;
80 +
81 + // Local temp variables
82 + size_t l, o, p;
83 + struct pageline *pgl = NULL;
84 +
85 + // --------------------------------------------------------------------
86 + // Startup: Init arch and open /proc/pagetypeinfo
87 + if (unlikely(!pagesize)) {
88 + pagesize = sysconf(_SC_PAGESIZE);
89 + }
90 +
91 + if(unlikely(!ff)) {
92 + snprintfz(ff_path, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME);
93 + ff = procfile_open(config_get(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "filename to monitor", ff_path), " \t:", PROCFILE_FLAG_DEFAULT);
94 +
95 + if(unlikely(!ff)) {
96 + strncpyz(ff_path, PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME, FILENAME_MAX);
97 + ff = procfile_open(PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME, " \t,", PROCFILE_FLAG_DEFAULT);
98 + }
99 + }
100 + if(unlikely(!ff))
101 + return 1;
102 +
103 + ff = procfile_readall(ff);
104 + if(unlikely(!ff))
105 + return 0; // we return 0, so that we will retry to open it next time
106 +
107 + // --------------------------------------------------------------------
108 + // Init: find how many Nodes, Zones and Types
109 + if(unlikely(pagelines_cnt == 0)) {
110 + size_t nodenumlast = -1;
111 + char *zonenamelast = NULL;
112 +
113 + ff_lines = procfile_lines(ff);
114 + if(unlikely(!ff_lines)) {
115 + error("PLUGIN: PROC_PAGETYPEINFO: Cannot read %s, zero lines reported.", ff_path);
116 + return 1;
117 + }
118 +
119 + // Configuration
120 + do_global = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "enable system summary", CONFIG_BOOLEAN_YES);
121 + do_detail = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "enable detail per-type", CONFIG_BOOLEAN_AUTO);
122 + filter_types = simple_pattern_create(
123 + config_get(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "hide charts id matching", "")
124 + , NULL
125 + , SIMPLE_PATTERN_SUFFIX
126 + );
127 +
128 + pagelines_cnt = 0;
129 +
130 + // Pass 1: how many lines would be valid
131 + for (l = 4; l < ff_lines; l++) {
132 + if (!pagetypeinfo_line_valid(ff, l))
133 + continue;
134 +
135 + pagelines_cnt++;
136 + }
137 + if (pagelines_cnt == 0) {
138 + error("PLUGIN: PROC_PAGETYPEINFO: Unable to parse any valid line in %s", ff_path);
139 + return 1;
140 + }
141 +
142 + // 4th line is the "Free pages count per migrate type at order". Just substract these 8 words.
143 + pageorders_cnt = procfile_linewords(ff, 3);
144 + if (pageorders_cnt < 9) {
145 + error("PLUGIN: PROC_PAGETYPEINFO: Unable to parse Line 4 of %s", ff_path);
146 + return 1;
147 + }
148 +
149 + pageorders_cnt -= 9;
150 +
151 + if (pageorders_cnt > MAX_PAGETYPE_ORDER) {
152 + error("PLUGIN: PROC_PAGETYPEINFO: pageorder found (%lu) is higher than max %d", pageorders_cnt, MAX_PAGETYPE_ORDER);
153 + return 1;
154 + }
155 +
156 + // Init pagelines from scanned lines
157 + if (!pagelines) {
158 + pagelines = callocz(pagelines_cnt, sizeof(struct pageline));
159 + if (!pagelines) {
160 + error("PLUGIN: PROC_PAGETYPEINFO: Cannot allocate %lu pagelines of %lu B", pagelines_cnt, sizeof(struct pageline));
161 + return 1;
162 + }
163 + }
164 +
165 + // Pass 2: Scan the file again, with details
166 + p = 0;
167 + for (l=4; l < ff_lines; l++) {
168 +
169 + if (!pagetypeinfo_line_valid(ff, l))
170 + continue;
171 +
172 + size_t nodenum = strtoul(procfile_lineword(ff, l, 1), NULL, 10);
173 + char *zonename = procfile_lineword(ff, l, 3);
174 + char *typename = procfile_lineword(ff, l, 5);
175 +
176 + // We changed node or zone
177 + if (nodenum != nodenumlast || !zonenamelast || strncmp(zonename, zonenamelast, 6) != 0) {
178 + zonenamelast = zonename;
179 + }
180 +
181 + // Populate the line
182 + pgl = &pagelines[p];
183 +
184 + pgl->line = l;
185 + pgl->node = nodenum;
186 + pgl->type = typename;
187 + pgl->zone = zonename;
188 + for (o = 0; o < pageorders_cnt; o++)
189 + pgl->free_pages_size[o] = str2uint64_t(procfile_lineword(ff, l, o+6)) * 1 << o;
190 +
191 + p++;
192 + }
193 +
194 + // Init the RRD graphs
195 +
196 + // Per-Order: sum of all node, zone, type Grouped by order
197 + if (do_global != CONFIG_BOOLEAN_NO) {
198 + st_order = rrdset_create_localhost(
199 + "mem"
200 + , "pagetype_global"
201 + , NULL
202 + , "pagetype"
203 + , NULL
204 + , "System orders available"
205 + , "B"
206 + , PLUGIN_PROC_NAME
207 + , PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME
208 + , NETDATA_CHART_PRIO_MEM_PAGEFRAG
209 + , update_every
210 + , RRDSET_TYPE_STACKED
211 + );
212 + for (o = 0; o < pageorders_cnt; o++) {
213 + char id[3+1];
214 + snprintfz(id, 3, "%lu", o);
215 +
216 + char name[20+1];
217 + dim_name(name, o, pagesize);
218 +
219 + systemorders[o].rd = rrddim_add(st_order, id, name, pagesize, 1, RRD_ALGORITHM_ABSOLUTE);
220 + }
221 + }
222 +
223 +
224 + // Per-Numa Node & Zone & Type (full detail). Only if sum(line) > 0
225 + st_nodezonetype = callocz(pagelines_cnt, sizeof(RRDSET));
226 + for (p = 0; p < pagelines_cnt; p++) {
227 + pgl = &pagelines[p];
228 +
229 + // Skip invalid, refused or empty pagelines if not explicitely requested
230 + if (!pgl
231 + || do_detail == CONFIG_BOOLEAN_NO
232 + || (do_detail == CONFIG_BOOLEAN_AUTO && pageline_total_count(pgl) == 0 && netdata_zero_metrics_enabled != CONFIG_BOOLEAN_YES))
233 + continue;
234 +
235 + // "pagetype Node" + NUMA-NodeId + ZoneName + TypeName
236 + char setid[13+1+2+1+MAX_ZONETYPE_NAME+1+MAX_PAGETYPE_NAME+1];
237 + snprintfz(setid, 13+1+2+1+MAX_ZONETYPE_NAME+1+MAX_PAGETYPE_NAME, "pagetype_Node%d_%s_%s", pgl->node, pgl->zone, pgl->type);
238 +
239 + // Skip explicitely refused charts
240 + if (simple_pattern_matches(filter_types, setid))
241 + continue;
242 +
243 + // "Node" + NUMA-NodeID + ZoneName + TypeName
244 + char setname[4+1+MAX_ZONETYPE_NAME+1+MAX_PAGETYPE_NAME +1];
245 + snprintfz(setname, MAX_ZONETYPE_NAME + MAX_PAGETYPE_NAME, "Node %d %s %s",
246 + pgl->node, pgl->zone, pgl->type);
247 +
248 + st_nodezonetype[p] = rrdset_create_localhost(
249 + "mem"
250 + , setid
251 + , NULL
252 + , "pagetype"
253 + , NULL
254 + , setname
255 + , "B"
256 + , PLUGIN_PROC_NAME
257 + , PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME
258 + , NETDATA_CHART_PRIO_MEM_PAGEFRAG + 1 + p
259 + , update_every
260 + , RRDSET_TYPE_STACKED
261 + );
262 + for (o = 0; o < pageorders_cnt; o++) {
263 + char dimid[3+1];
264 + snprintfz(dimid, 3, "%lu", o);
265 + char dimname[20+1];
266 + dim_name(dimname, o, pagesize);
267 +
268 + pgl->rd[o] = rrddim_add(st_nodezonetype[p], dimid, dimname, pagesize, 1, RRD_ALGORITHM_ABSOLUTE);
269 + }
270 + }
271 + }
272 +
273 + // --------------------------------------------------------------------
274 + // Update pagelines
275 +
276 + // Process each line
277 + p = 0;
278 + for (l=4; l<ff_lines; l++) {
279 +
280 + if (!pagetypeinfo_line_valid(ff, l))
281 + continue;
282 +
283 + size_t words = procfile_linewords(ff, l);
284 +
285 + if (words != 7+pageorders_cnt) {
286 + error("PLUGIN: PROC_PAGETYPEINFO: Unable to read line %lu, %lu words found instead of %lu", l+1, words, 7+pageorders_cnt);
287 + break;
288 + }
289 +
290 + for (o = 0; o < pageorders_cnt; o++) {
291 + // Reset counter
292 + if (p == 0)
293 + systemorders[o].size = 0;
294 +
295 + // Update orders of the current line
296 + pagelines[p].free_pages_size[o] = str2uint64_t(procfile_lineword(ff, l, o+6)) * 1 << o;
297 +
298 + // Update sum by order
299 + systemorders[o].size += pagelines[p].free_pages_size[o];
300 + }
301 +
302 + p++;
303 + }
304 +
305 + // --------------------------------------------------------------------
306 + // update RRD values
307 +
308 + // Global system per order
309 + if (st_order) {
310 + rrdset_next(st_order);
311 + for (o = 0; o < pageorders_cnt; o++) {
312 + rrddim_set_by_pointer(st_order, systemorders[o].rd, systemorders[o].size);
313 + }
314 + rrdset_done(st_order);
315 + }
316 +
317 + // Per Node-Zone-Type
318 + if (do_detail) {
319 + for (p = 0; p < pagelines_cnt; p++) {
320 + // Skip empty graphs
321 + if (!st_nodezonetype[p])
322 + continue;
323 +
324 + rrdset_next(st_nodezonetype[p]);
325 + for (o = 0; o < pageorders_cnt; o++)
326 + rrddim_set_by_pointer(st_nodezonetype[p], pagelines[p].rd[o], pagelines[p].free_pages_size[o]);
327 +
328 + rrdset_done(st_nodezonetype[p]);
329 + }
330 + }
331 +
332 + return 0;
333 +}