replace GA with PostHog for backend telemetry events. (#10636)
- update anonymous-statistics.sh.in to use Netdata PostHog instead of GA - update docs accordingly
Andrew Maguire committed
Mar 10, 2021 at 16:18 UTC
6aeeb63eca08e85450434f453b9379d02255a343
2 files changed
+90
-109
daemon/anonymous-statistics.sh.in
+60
-73
@@ -26,79 +26,66 @@ fi
26
NETDATA_VERSION=$(echo "${NETDATA_VERSION}" | sed 's/-.*//g' | tr -d 'v')
27
28
# -------------------------------------------------------------------------------------------------
29
-# send the anonymous statistics to GA
30
-# https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters
31
-# The maximum index for a cd parameter is 20 so we have effectively run out.
29
+
30
+# define body of request to be sent
31
+REQ_BODY="$(cat << EOF
32
+{
33
+ "api_key": "mqkwGT0JNFqO-zX2t0mW6Tec9yooaVu7xCBlXtHnt5Y",
34
+ "event": "${ACTION} ${ACTION_RESULT}",
35
+ "properties": {
36
+ "distinct_id": "${NETDATA_REGISTRY_UNIQUE_ID}",
37
+ "\$current_url": "agent backend",
38
+ "\$pathname": "netdata-backend",
39
+ "\$host": "backend.netdata.io",
40
+ "\$ip": "127.0.0.1",
41
+ "event_source": "agent backend",
42
+ "action": "${ACTION}",
43
+ "action_result": "${ACTION_RESULT}",
44
+ "action_data": "${ACTION_DATA}",
45
+ "netdata_machine_guid": "${NETDATA_REGISTRY_UNIQUE_ID}",
46
+ "netdata_version": "${NETDATA_VERSION}",
47
+ "host_os_name": "${NETDATA_HOST_OS_NAME}",
48
+ "host_os_id": "${NETDATA_HOST_OS_ID}",
49
+ "host_os_id_like": "${NETDATA_HOST_OS_ID_LIKE}",
50
+ "host_os_version": "${NETDATA_HOST_OS_VERSION}",
51
+ "host_os_version_id": "${NETDATA_HOST_OS_VERSION_ID}",
52
+ "host_os_detection": "${NETDATA_HOST_OS_DETECTION}",
53
+ "host_is_k8s_node": "${NETDATA_HOST_IS_K8S_NODE}",
54
+ "system_kernel_name": "${NETDATA_SYSTEM_KERNEL_NAME}",
55
+ "system_kernel_version": "${NETDATA_SYSTEM_KERNEL_VERSION}",
56
+ "system_architecture": "${NETDATA_SYSTEM_ARCHITECTURE}",
57
+ "system_virtualization": "${NETDATA_SYSTEM_VIRTUALIZATION}",
58
+ "system_virt_detection": "${NETDATA_SYSTEM_VIRT_DETECTION}",
59
+ "system_container": "${NETDATA_SYSTEM_CONTAINER}",
60
+ "system_container_detection": "${NETDATA_SYSTEM_CONTAINER_DETECTION}",
61
+ "container_os_name": "${NETDATA_CONTAINER_OS_NAME}",
62
+ "container_os_id": "${NETDATA_CONTAINER_OS_ID}",
63
+ "container_os_id_like": "${NETDATA_CONTAINER_OS_ID_LIKE}",
64
+ "container_os_version": "${NETDATA_CONTAINER_OS_VERSION}",
65
+ "container_os_version_id": "${NETDATA_CONTAINER_OS_VERSION_ID}",
66
+ "container_os_detection": "${NETDATA_CONTAINER_OS_DETECTION}",
67
+ "system_cpu_detection": "${NETDATA_SYSTEM_CPU_DETECTION}",
68
+ "system_cpu_freq": "${NETDATA_SYSTEM_CPU_FREQ}",
69
+ "system_cpu_logical_cpu_count": "${NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT}",
70
+ "system_cpu_model": "${NETDATA_SYSTEM_CPU_MODEL}",
71
+ "system_cpu_vendor": "${NETDATA_SYSTEM_CPU_VENDOR}",
72
+ "system_disk_detection": "${NETDATA_SYSTEM_DISK_DETECTION}",
73
+ "system_ram_detection": "${NETDATA_SYSTEM_RAM_DETECTION}",
74
+ "system_total_disk_size": "${NETDATA_SYSTEM_TOTAL_DISK_SIZE}",
75
+ "system_total_ram": "${NETDATA_SYSTEM_TOTAL_RAM}"
76
+ }
77
+}
78
+EOF
79
+)"
80
+
81
+# send the anonymous statistics to the Netdata PostHog
82
if [ -n "$(command -v curl 2> /dev/null)" ]; then
33
- curl -X POST -Ss --max-time 2 \
34
- --data "v=1" \
35
- --data "tid=UA-64295674-3" \
36
- --data "aip=1" \
37
- --data "ds=shell" \
38
- --data-urlencode "cid=${NETDATA_REGISTRY_UNIQUE_ID}" \
39
- --data-urlencode "cs=${NETDATA_REGISTRY_UNIQUE_ID}" \
40
- --data "t=event" \
41
- --data "ni=1" \
42
- --data "an=anonymous-statistics" \
43
- --data-urlencode "av=${NETDATA_VERSION}" \
44
- --data-urlencode "ec=${ACTION}" \
45
- --data-urlencode "ea=${ACTION_RESULT}" \
46
- --data-urlencode "el=${ACTION_DATA}" \
47
- --data-urlencode "cd1=${NETDATA_HOST_OS_NAME}" \
48
- --data-urlencode "cd2=${NETDATA_HOST_OS_ID}" \
49
- --data-urlencode "cd3=${NETDATA_HOST_OS_ID_LIKE}" \
50
- --data-urlencode "cd4=${NETDATA_HOST_OS_VERSION}" \
51
- --data-urlencode "cd5=${NETDATA_HOST_OS_VERSION_ID}" \
52
- --data-urlencode "cd6=${NETDATA_HOST_OS_DETECTION}" \
53
- --data-urlencode "cd7=${NETDATA_SYSTEM_KERNEL_NAME}" \
54
- --data-urlencode "cd8=${NETDATA_SYSTEM_KERNEL_VERSION}" \
55
- --data-urlencode "cd9=${NETDATA_SYSTEM_ARCHITECTURE}" \
56
- --data-urlencode "cd10=${NETDATA_SYSTEM_VIRTUALIZATION}" \
57
- --data-urlencode "cd11=${NETDATA_SYSTEM_VIRT_DETECTION}" \
58
- --data-urlencode "cd12=${NETDATA_SYSTEM_CONTAINER}" \
59
- --data-urlencode "cd13=${NETDATA_SYSTEM_CONTAINER_DETECTION}" \
60
- --data-urlencode "cd14=${NETDATA_CONTAINER_OS_NAME}" \
61
- --data-urlencode "cd15=${NETDATA_CONTAINER_OS_ID}" \
62
- --data-urlencode "cd16=${NETDATA_CONTAINER_OS_ID_LIKE}" \
63
- --data-urlencode "cd17=${NETDATA_CONTAINER_OS_VERSION}" \
64
- --data-urlencode "cd18=${NETDATA_CONTAINER_OS_VERSION_ID}" \
65
- --data-urlencode "cd19=${NETDATA_CONTAINER_OS_DETECTION}" \
66
- --data-urlencode "cd20=${NETDATA_HOST_IS_K8S_NODE}" \
67
- "https://www.google-analytics.com/collect" > /dev/null 2>&1
83
+ curl -X POST --header "Content-Type: application/json" -d "${REQ_BODY}" https://posthog.netdata.cloud/capture/ > /dev/null 2>&1
84
else
69
- wget -q -O - --timeout=1 "https://www.google-analytics.com/collect?\
70
-&v=1\
71
-&tid=UA-64295674-3\
72
-&aip=1\
73
-&ds=shell\
74
-&cid=${NETDATA_REGISTRY_UNIQUE_ID}\
75
-&cs=${NETDATA_REGISTRY_UNIQUE_ID}\
76
-&t=event\
77
-&ni=1\
78
-&an=anonymous-statistics\
79
-&av=${NETDATA_VERSION}\
80
-&ec=${ACTION}\
81
-&ea=${ACTION_RESULT}\
82
-&el=${ACTION_DATA}\
83
-&cd1=${NETDATA_HOST_OS_NAME}\
84
-&cd2=${NETDATA_HOST_OS_ID}\
85
-&cd3=${NETDATA_HOST_OS_ID_LIKE}\
86
-&cd4=${NETDATA_HOST_OS_VERSION}\
87
-&cd5=${NETDATA_HOST_OS_VERSION_ID}\
88
-&cd6=${NETDATA_HOST_OS_DETECTION}\
89
-&cd7=${NETDATA_SYSTEM_KERNEL_NAME}\
90
-&cd8=${NETDATA_SYSTEM_KERNEL_VERSION}\
91
-&cd9=${NETDATA_SYSTEM_ARCHITECTURE}\
92
-&cd10=${NETDATA_SYSTEM_VIRTUALIZATION}\
93
-&cd11=${NETDATA_SYSTEM_VIRT_DETECTION}\
94
-&cd12=${NETDATA_SYSTEM_CONTAINER}\
95
-&cd13=${NETDATA_SYSTEM_CONTAINER_DETECTION}\
96
-&cd14=${NETDATA_CONTAINER_OS_NAME} \
97
-&cd15=${NETDATA_CONTAINER_OS_ID} \
98
-&cd16=${NETDATA_CONTAINER_OS_ID_LIKE} \
99
-&cd17=${NETDATA_CONTAINER_OS_VERSION} \
100
-&cd18=${NETDATA_CONTAINER_OS_VERSION_ID} \
101
-&cd19=${NETDATA_CONTAINER_OS_DETECTION} \
102
-&cd20=${NETDATA_HOST_IS_K8S_NODE} \
103
-" > /dev/null 2>&1
85
+ wget -q -O - --no-check-certificate \
86
+ --method POST \
87
+ --timeout=1 \
88
+ --header 'Content-Type: application/json' \
89
+ --body-data "${REQ_BODY}" \
90
+ 'https://posthog.netdata.cloud/capture/' > /dev/null 2>&1
91
fi
docs/anonymous-statistics.md
+30
-36
@@ -1,13 +1,13 @@
1
<!--
2
+---
3
title: "Anonymous statistics"
3
-description: "The Netdata Agent collects anonymous usage information by default and sends it to Google Analytics for quality assurance and product decisions."
4
custom_edit_url: https://github.com/netdata/netdata/edit/master/docs/anonymous-statistics.md
5
+---
6
-->
7
8
# Anonymous statistics
9
9
-Starting with v1.12, Netdata collects anonymous usage information by default and sends it to Google Analytics. We use
10
-the statistics gathered from this information for two purposes:
10
+Netdata collects anonymous usage information by default using the open-source product analytics platform [PostHog](https://github.com/PostHog/posthog). We self-host our PostHog instance, which means your data is never sent or processed by any third parties outside of the Netdata infrastructure. We use the statistics gathered from this information for two purposes:
11
12
1. **Quality assurance**, to help us understand if Netdata behaves as expected, and to help us classify repeated
13
issues with certain distributions or environments.
@@ -15,49 +15,38 @@ the statistics gathered from this information for two purposes:
15
2. **Usage statistics**, to help us interpret how people use the Netdata agent in real-world environments, and to help
16
us identify how our development/design decisions influence the community.
17
18
-Netdata sends information to Google Analytics via two different channels:
18
+Netdata collects usage information via two different channels:
19
20
-- Google Tag Manager fires when you access an agent's dashboard.
21
-- The Netdata daemon executes the [`anonymous-statistics.sh`
22
- script](https://github.com/netdata/netdata/blob/6469cf92724644f5facf343e4bdd76ac0551a418/daemon/anonymous-statistics.sh.in)
23
- when Netdata starts, stops cleanly, or fails.
20
+- **Agent dashboard**: We use the [PostHog JavaScript integration](https://posthog.com/docs/integrations/js-integration) (with sensitive event attributes overwritten to be anonymized) to send product usage events when you access an [Agent's dashboard](/web/gui/README.md).
21
+- **Agent backend**: The `netdata` daemon executes the [`anonymous-statistics.sh`](https://github.com/netdata/netdata/blob/6469cf92724644f5facf343e4bdd76ac0551a418/daemon/anonymous-statistics.sh.in) script when Netdata starts, stops cleanly, or fails.
22
23
You can opt-out from sending anonymous statistics to Netdata through three different [opt-out mechanisms](#opt-out).
24
27
-## Google tag manager
25
+## Agent Dashboard - PostHog JavaScript
26
29
-Google tag manager (GTM) is the recommended way of collecting statistics for new implementations using GA. Unlike the
30
-older API, the logic of when to send information to GA and what information to send is controlled centrally.
27
+When you kick off an Agent dashboard session by visiting `http://NODE:19999`, Netdata will initialiszes a PostHog session and masks various event attributes.
28
32
-We have configured GTM to trigger the tag only when the variable `anonymous_statistics` is true. The value of this
33
-variable is controlled via the [opt-out mechanism](#opt-out).
34
-
35
-To ensure anonymity of the stored information, we have configured GTM's GA variable "Fields to set" as follows:
29
+_Note_: You can see the relevant code in the [dashboard repository](https://github.com/netdata/dashboard/blob/master/src/domains/global/sagas.ts#L107) where the `window.posthog.register()` call is made.
30
37
-| Field name | Value |
38
-| -------------- | -------------------------------------------------- |
39
-| page | netdata-dashboard |
40
-| hostname | dashboard.my-netdata.io |
41
-| anonymizeIp | true |
42
-| title | Netdata dashboard |
43
-| campaignSource | {{machine_guid}} |
44
-| campaignMedium | web |
45
-| referrer | <http://dashboard.my-netdata.io> |
46
-| Page URL | <http://dashboard.my-netdata.io/netdata-dashboard> |
47
-| Page Hostname | <http://dashboard.my-netdata.io> |
48
-| Page Path | /netdata-dashboard |
49
-| location | <http://dashboard.my-netdata.io> |
31
+```JavaScript
32
+window.posthog.register({
33
+ distinct_id: machineGuid,
34
+ $ip: "127.0.0.1",
35
+ $current_url: "agent dashboard",
36
+ $pathname: "netdata-dashboard",
37
+ $host: "dashboard.netdata.io",
38
+})
39
+```
40
51
-In addition, the Netdata-generated unique machine guid is sent to GA via a custom dimension.
52
-You can verify the effect of these settings by examining the GA `collect` request parameters.
41
+In the above snippet a Netdata PostHog session is initialized and the `ip`, `current_url`, `pathname` and `host` attributes are set to constant values for all events that may be sent during the session. This way, information like the IP or hostname of the Agent will not be sent as part of the product usage event data.
42
54
-The only thing that's impossible for us to prevent from being **sent** is the URL in the "Referrer" Header of the
55
-browser request to GA. However, the settings above ensure that all **stored** URLs and host names are anonymized.
43
+We have configured the dashboard to trigger the PostHog JavaScript code only when the variable `anonymous_statistics` is true. The value of this
44
+variable is controlled via the [opt-out mechanism](#opt-out).
45
57
-## Anonymous Statistics Script
46
+## Agent Backend - Anonymous Statistics Script
47
48
Every time the daemon is started or stopped and every time a fatal condition is encountered, Netdata uses the anonymous
60
-statistics script to collect system information and send it to GA via an http call. The information collected for all
49
+statistics script to collect system information and send it to the Netdata PostHog via an http call. The information collected for all
50
events is:
51
52
- Netdata version
@@ -103,7 +92,12 @@ Each of these opt-out processes does the following:
92
93
- Prevents the daemon from executing the anonymous statistics script.
94
- Forces the anonymous statistics script to exit immediately.
106
-- Stops the Google Tag Manager Javascript snippet, which remains on the dashboard, from firing and sending any data to
107
- Google Analytics.
95
+- Stops the PostHog Javascript snippet, which remains on the dashboard, from firing and sending any data to the Netdata PostHog.
96
+
97
+## Migration from Google Analytics and Google Tag Manager.
98
+
99
+Prior to v1.29.4 we used Google Analytics to capture this information. This led to discomfort with some of our users in sending any product usage data to a third party like Google. It was also not even that useful in terms of generating the insights we needed to help catch bugs early and find opportunities for product improvement as Google Analytics does not allow its users access to the raw underlying data without paying a significant amount of money which would be infeasible for a project like Netdata.
100
+
101
+While we migrate fully away from Google Analytics to PostHog there maybe be a small period of time where we run both in parallel before we remove all Google Analytics related code. This is to ensure we can fully test and validate the Netdata PostHog implementation before fully defaulting to it.
102
103
[]()