Ιmplementation to add logs integrations (#18791)
Fotis Voutsas committed
Oct 16, 2024 at 14:37 UTC
dfcf47e4ccda35c1d3efd11328bfa6b01b4b1319
7 files changed
+338
-14
integrations/gen_docs_integrations.py
+63
@@ -25,6 +25,9 @@ def cleanup():
25
for element in Path("integrations/cloud-notifications").glob('**/*/'):
26
if "integrations" in str(element) and not "metadata.yaml" in str(element):
27
shutil.rmtree(element)
28
+ for element in Path("integrations/logs").glob('**/*/'):
29
+ if "integrations" in str(element) and "metadata.yaml" not in str(element):
30
+ shutil.rmtree(element)
31
for element in Path("integrations/cloud-authentication").glob('**/*/'):
32
if "integrations" in str(element) and not "metadata.yaml" in str(element):
33
shutil.rmtree(element)
@@ -76,6 +79,7 @@ def add_custom_edit_url(markdown_string, meta_yaml_link, sidebar_label_string, m
79
"""
80
81
output = ""
82
+ path_to_md_file = ""
83
84
if mode == 'default':
85
path_to_md_file = f'{meta_yaml_link.replace("/metadata.yaml", "")}/integrations/{clean_string(sidebar_label_string)}'
@@ -86,6 +90,9 @@ def add_custom_edit_url(markdown_string, meta_yaml_link, sidebar_label_string, m
90
elif mode == 'agent-notification':
91
path_to_md_file = meta_yaml_link.replace("metadata.yaml", "README")
92
93
+ elif mode == 'logs':
94
+ path_to_md_file = meta_yaml_link.replace("metadata.yaml", "README")
95
+
96
elif mode == 'cloud-authentication':
97
path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
98
@@ -293,6 +300,34 @@ endmeta-->
300
except Exception as e:
301
print("Exception in notification md construction", e, integration['id'])
302
303
+ elif mode == 'logs':
304
+ try:
305
+ # initiate the variables for the logs integration
306
+ meta_yaml = integration['edit_link'].replace("blob", "edit")
307
+ sidebar_label = integration['meta']['name']
308
+ learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
309
+
310
+ # build the markdown string
311
+ md = \
312
+ f"""<!--startmeta
313
+meta_yaml: "{meta_yaml}"
314
+sidebar_label: "{sidebar_label}"
315
+learn_status: "Published"
316
+learn_rel_path: "{learn_rel_path.replace("logs", "Logs")}"
317
+message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE LOGS' metadata.yaml FILE"
318
+endmeta-->
319
+
320
+{create_overview(integration, integration['meta']['icon_filename'])}"""
321
+
322
+ if integration['setup']:
323
+ md += f"""
324
+{integration['setup']}
325
+"""
326
+
327
+ except Exception as e:
328
+ print("Exception in logs md construction", e, integration['id'])
329
+
330
+
331
# AUTHENTICATIONS
332
elif mode == 'authentication':
333
if True:
@@ -417,7 +452,28 @@ def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default')
452
453
except FileNotFoundError as e:
454
print("Exception in writing to file", e)
455
+ elif mode == 'logs':
456
+
457
+ name = clean_string(integration['meta']['name'])
458
+
459
+ if not Path(f'{path}/integrations').exists():
460
+ Path(f'{path}/integrations').mkdir()
461
+
462
+ # proper_edit_name = meta_yaml.replace(
463
+ # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
464
+
465
+ md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='logs')
466
+
467
+ finalpath = f'{path}/integrations/{name}.md'
468
469
+ try:
470
+ clean_and_write(
471
+ md,
472
+ Path(finalpath)
473
+ )
474
+
475
+ except FileNotFoundError as e:
476
+ print("Exception in writing to file", e)
477
elif mode == 'authentication':
478
479
name = clean_string(integration['meta']['name'])
@@ -503,6 +559,13 @@ for integration in integrations:
559
path = build_path(meta_yaml)
560
write_to_file(path, md, meta_yaml, sidebar_label, community, mode='cloud-notification')
561
562
+ elif integration['integration_type'] == "logs":
563
+
564
+ meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
565
+ integration, mode='logs')
566
+ path = build_path(meta_yaml)
567
+ write_to_file(path, md, meta_yaml, sidebar_label, community, mode='logs')
568
+
569
elif integration['integration_type'] == "authentication":
570
571
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
integrations/gen_integrations.py
+116
-14
@@ -47,6 +47,10 @@ CLOUD_NOTIFICATION_SOURCES = [
47
(AGENT_REPO, INTEGRATIONS_PATH / 'cloud-notifications' / 'metadata.yaml', False),
48
]
49
50
+LOGS_SOURCES = [
51
+ (AGENT_REPO, INTEGRATIONS_PATH / 'logs' / 'metadata.yaml', False),
52
+]
53
+
54
AUTHENTICATION_SOURCES = [
55
(AGENT_REPO, INTEGRATIONS_PATH / 'cloud-authentication' / 'metadata.yaml', False),
56
]
@@ -77,6 +81,11 @@ CLOUD_NOTIFICATION_RENDER_KEYS = [
81
'troubleshooting',
82
]
83
84
+LOGS_RENDER_KEYS = [
85
+ 'overview',
86
+ 'setup',
87
+]
88
+
89
AUTHENTICATION_RENDER_KEYS = [
90
'overview',
91
'setup',
@@ -139,6 +148,11 @@ CLOUD_NOTIFICATION_VALIDATOR = Draft7Validator(
148
registry=registry,
149
)
150
151
+LOGS_VALIDATOR = Draft7Validator(
152
+ {'$ref': './logs.json#'},
153
+ registry=registry,
154
+)
155
+
156
AUTHENTICATION_VALIDATOR = Draft7Validator(
157
{'$ref': './authentication.json#'},
158
registry=registry,
@@ -399,6 +413,19 @@ def _load_agent_notification_file(file, repo):
413
return ret
414
415
416
+def load_agent_notifications():
417
+ ret = []
418
+
419
+ for repo, path, match in AGENT_NOTIFICATION_SOURCES:
420
+ if match and path.exists() and path.is_dir():
421
+ for file in path.glob(METADATA_PATTERN):
422
+ ret.extend(_load_agent_notification_file(file, repo))
423
+ elif not match and path.exists() and path.is_file():
424
+ ret.extend(_load_agent_notification_file(path, repo))
425
+
426
+ return ret
427
+
428
+
429
def _load_cloud_notification_file(file, repo):
430
debug(f'Loading {file}.')
431
data = load_yaml(file)
@@ -432,28 +459,61 @@ def _load_cloud_notification_file(file, repo):
459
return ret
460
461
435
-def load_agent_notifications():
462
+def load_cloud_notifications():
463
ret = []
464
438
- for repo, path, match in AGENT_NOTIFICATION_SOURCES:
465
+ for repo, path, match in CLOUD_NOTIFICATION_SOURCES:
466
if match and path.exists() and path.is_dir():
467
for file in path.glob(METADATA_PATTERN):
441
- ret.extend(_load_agent_notification_file(file, repo))
468
+ ret.extend(_load_cloud_notification_file(file, repo))
469
elif not match and path.exists() and path.is_file():
443
- ret.extend(_load_agent_notification_file(path, repo))
470
+ ret.extend(_load_cloud_notification_file(path, repo))
471
472
return ret
473
474
448
-def load_cloud_notifications():
475
+def _load_logs_file(file, repo):
476
+ debug(f'Loading {file}.')
477
+ data = load_yaml(file)
478
+
479
+ if not data:
480
+ return []
481
+
482
+ try:
483
+ LOGS_VALIDATOR.validate(data)
484
+ except ValidationError:
485
+ warn(f'Failed to validate {file} against the schema.', file)
486
+ return []
487
+
488
+ if 'id' in data:
489
+ data['integration_type'] = 'logs'
490
+ data['_src_path'] = file
491
+ data['_repo'] = repo
492
+ data['_index'] = 0
493
+
494
+ return [data]
495
+ else:
496
+ ret = []
497
+
498
+ for idx, item in enumerate(data):
499
+ item['integration_type'] = 'logs'
500
+ item['_src_path'] = file
501
+ item['_repo'] = repo
502
+ item['_index'] = idx
503
+ ret.append(item)
504
+
505
+ return ret
506
+
507
+
508
+def load_logs():
509
ret = []
510
451
- for repo, path, match in CLOUD_NOTIFICATION_SOURCES:
511
+ for repo, path, match in LOGS_SOURCES:
512
if match and path.exists() and path.is_dir():
513
for file in path.glob(METADATA_PATTERN):
454
- ret.extend(_load_cloud_notification_file(file, repo))
514
+ ret.extend(_load_logs_file(file, repo))
515
elif not match and path.exists() and path.is_file():
456
- ret.extend(_load_cloud_notification_file(path, repo))
516
+ ret.extend(_load_logs_file(path, repo))
517
518
return ret
519
@@ -818,6 +878,48 @@ def render_cloud_notifications(categories, notifications, ids):
878
return notifications, clean_notifications, ids
879
880
881
+def render_logs(categories, logs, ids):
882
+ debug('Sorting logs.')
883
+
884
+ sort_integrations(logs)
885
+
886
+ debug('Checking log ids.')
887
+
888
+ logs, ids = dedupe_integrations(logs, ids)
889
+
890
+ clean_logs = []
891
+
892
+ for item in logs:
893
+ item['edit_link'] = make_edit_link(item)
894
+
895
+ clean_item = deepcopy(item)
896
+
897
+ for key in LOGS_RENDER_KEYS:
898
+ if key in item.keys():
899
+ template = get_jinja_env().get_template(f'{key}.md')
900
+ data = template.render(entry=item, clean=False)
901
+ clean_data = template.render(entry=item, clean=True)
902
+
903
+ if 'variables' in item['meta']:
904
+ template = get_jinja_env().from_string(data)
905
+ data = template.render(variables=item['meta']['variables'], clean=False)
906
+ template = get_jinja_env().from_string(clean_data)
907
+ clean_data = template.render(variables=item['meta']['variables'], clean=True)
908
+ else:
909
+ data = ''
910
+ clean_data = ''
911
+
912
+ item[key] = data
913
+ clean_item[key] = clean_data
914
+
915
+ for k in ['_src_path', '_repo', '_index']:
916
+ del item[k], clean_item[k]
917
+
918
+ clean_logs.append(clean_item)
919
+
920
+ return logs, clean_logs, ids
921
+
922
+
923
def render_authentications(categories, authentications, ids):
924
debug('Sorting authentications.')
925
@@ -885,21 +987,21 @@ def main():
987
exporters = load_exporters()
988
agent_notifications = load_agent_notifications()
989
cloud_notifications = load_cloud_notifications()
990
+ logs = load_logs()
991
authentications = load_authentications()
992
993
collectors, clean_collectors, ids = render_collectors(categories, collectors, dict())
994
deploy, clean_deploy, ids = render_deploy(distros, categories, deploy, ids)
995
exporters, clean_exporters, ids = render_exporters(categories, exporters, ids)
893
- agent_notifications, clean_agent_notifications, ids = render_agent_notifications(categories, agent_notifications,
894
- ids)
895
- cloud_notifications, clean_cloud_notifications, ids = render_cloud_notifications(categories, cloud_notifications,
896
- ids)
996
+ agent_notifications, clean_agent_notifications, ids = render_agent_notifications(categories, agent_notifications,ids)
997
+ cloud_notifications, clean_cloud_notifications, ids = render_cloud_notifications(categories, cloud_notifications,ids)
998
+ logs, clean_logs, ids = render_logs(categories, logs,ids)
999
authentications, clean_authentications, ids = render_authentications(categories, authentications, ids)
1000
899
- integrations = collectors + deploy + exporters + agent_notifications + cloud_notifications + authentications
1001
+ integrations = collectors + deploy + exporters + agent_notifications + cloud_notifications + logs + authentications
1002
render_integrations(categories, integrations)
1003
902
- clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_agent_notifications + clean_cloud_notifications + clean_authentications
1004
+ clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_agent_notifications + clean_cloud_notifications + clean_logs + clean_authentications
1005
render_json(categories, clean_integrations)
1006
1007
integrations/logs/metadata.yaml
new
+38
@@ -0,0 +1,38 @@
1
+# yamllint disable rule:line-length
2
+---
3
+- id: "logs-systemd-journal"
4
+ meta:
5
+ name: "Systemd Journal Logs"
6
+ link: "https://github.com/netdata/netdata/blob/master/src/collectors/systemd-journal.plugin/README.md"
7
+ categories:
8
+ - logs
9
+ icon_filename: "netdata.png"
10
+ keywords:
11
+ - systemd
12
+ - journal
13
+ - logs
14
+ overview:
15
+ description: |
16
+ The `systemd` journal plugin by Netdata makes viewing, exploring and analyzing `systemd` journal logs simple and efficient.
17
+
18
+ It automatically discovers available journal sources, allows advanced filtering, offers interactive visual representations and supports exploring the logs of both individual servers and the logs on infrastructure wide journal centralization servers.
19
+
20
+ The plugin automatically detects the available journal sources, based on the journal files available in `/var/log/journal` (persistent logs) and `/run/log/journal` (volatile logs).
21
+ visualization:
22
+ description: |
23
+ You can start exploring `systemd` journal logs on the "Logs" tab of the Netdata UI.
24
+ key_features:
25
+ description: |
26
+ - Works on both **individual servers** and **journal centralization servers**.
27
+ - Supports `persistent` and `volatile` journals.
28
+ - Supports `system`, `user`, `namespaces` and `remote` journals.
29
+ - Allows filtering on **any journal field** or **field value**, for any time-frame.
30
+ - Allows **full text search** (`grep`) on all journal fields, for any time-frame.
31
+ - Provides a **histogram** for log entries over time, with a break down per field-value, for any field and any time-frame.
32
+ - Works directly on journal files, without any other third-party components.
33
+ - Supports coloring log entries, the same way `journalctl` does.
34
+ - In PLAY mode provides the same experience as `journalctl -f`, showing new log entries immediately after they are received.
35
+ setup:
36
+ prerequisites:
37
+ description: |
38
+ - A Netdata Cloud account
integrations/schemas/logs.json
new
+97
@@ -0,0 +1,97 @@
1
+{
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "title": "Netdata Logs integrations metadata.",
4
+ "oneOf": [
5
+ {
6
+ "$ref": "#/$defs/entry"
7
+ },
8
+ {
9
+ "type": "array",
10
+ "minLength": 1,
11
+ "items": {
12
+ "$ref": "#/$defs/entry"
13
+ }
14
+ }
15
+ ],
16
+ "$defs": {
17
+ "entry": {
18
+ "type": "object",
19
+ "description": "Data for a single logs integration.",
20
+ "properties": {
21
+ "id": {
22
+ "$ref": "./shared.json#/$defs/id"
23
+ },
24
+ "meta": {
25
+ "$ref": "./shared.json#/$defs/instance"
26
+ },
27
+ "keywords": {
28
+ "$ref": "./shared.json#/$defs/keywords"
29
+ },
30
+ "overview": {
31
+ "type": "object",
32
+ "properties": {
33
+ "description": {
34
+ "type": "string",
35
+ "description": "General description of what the integration does."
36
+ },
37
+ "visualization": {
38
+ "type": "object",
39
+ "properties": {
40
+ "description": {
41
+ "type": "string",
42
+ "description": "How the user can access the data provided by the integration"
43
+ }
44
+ },
45
+ "required": [
46
+ "description"
47
+ ]
48
+ },
49
+ "key_features": {
50
+ "type": "object",
51
+ "properties": {
52
+ "description": {
53
+ "type": "string",
54
+ "description": "The key features of the integration."
55
+ }
56
+ },
57
+ "required": [
58
+ "description"
59
+ ]
60
+ }
61
+ },
62
+ "required": [
63
+ "description",
64
+ "visualization",
65
+ "key_features"
66
+ ]
67
+ },
68
+ "setup": {
69
+ "type": "object",
70
+ "properties": {
71
+ "prerequisites": {
72
+ "type": "object",
73
+ "properties": {
74
+ "description": {
75
+ "type": "string",
76
+ "description": "Prerequisites of getting the integration working. For Log Functions only a Netdata account should be needed."
77
+ }
78
+ },
79
+ "required": [
80
+ "description"
81
+ ]
82
+ },
83
+ "required": [
84
+ "prerequisites"
85
+ ]
86
+ }
87
+ }
88
+ },
89
+ "required": [
90
+ "id",
91
+ "meta",
92
+ "keywords",
93
+ "overview"
94
+ ]
95
+ }
96
+ }
97
+}
\ No newline at end of file
integrations/templates/overview.md
+2
@@ -6,4 +6,6 @@
6
[% include 'overview/notification.md' %]
7
[% elif entry.integration_type == 'authentication' %]
8
[% include 'overview/authentication.md' %]
9
+[% elif entry.integration_type == 'logs' %]
10
+[% include 'overview/logs.md' %]
11
[% endif %]
integrations/templates/overview/logs.md
new
+11
@@ -0,0 +1,11 @@
1
+# [[ entry.meta.name ]]
2
+
3
+[[ entry.overview.description ]]
4
+
5
+## Visualization
6
+
7
+[[ entry.overview.visualization.description ]]
8
+
9
+## Key features
10
+
11
+[[ entry.overview.key_features.description ]]
\ No newline at end of file
integrations/templates/setup.md
+11
@@ -1,4 +1,14 @@
1
## Setup
2
+[% if entry.integration_type == 'logs' %]
3
+
4
+## Prerequisites
5
+
6
+[[ entry.setup.prerequisites.description]]
7
+
8
+## Configuration
9
+
10
+There is no configuration needed for this integration.
11
+[% else %]
12
13
[% if entry.setup.description %]
14
[[ entry.setup.description ]]
@@ -106,3 +116,4 @@ There are no configuration examples.
116
117
[% endif %]
118
[% endif %]
119
+[% endif %]
\ No newline at end of file