| 1 | # yamllint disable rule:line-length |
| 2 | --- |
| 3 | id: 'export-google-pubsub' |
| 4 | meta: |
| 5 | name: 'Google Cloud Pub Sub' |
| 6 | link: 'https://cloud.google.com/pubsub' |
| 7 | categories: |
| 8 | - export |
| 9 | icon_filename: 'pubsub.png' |
| 10 | keywords: |
| 11 | - exporter |
| 12 | - Google Cloud |
| 13 | - Pub Sub |
| 14 | overview: |
| 15 | exporter_description: | |
| 16 | Export metrics to Google Cloud Pub/Sub Service |
| 17 | exporter_limitations: '' |
| 18 | setup: |
| 19 | prerequisites: |
| 20 | list: |
| 21 | - title: '' |
| 22 | description: | |
| 23 | - First [install](https://github.com/googleapis/google-cloud-cpp/) install Google Cloud Platform C++ Client Libraries |
| 24 | - Pub/Sub support is also dependent on the dependencies of those libraries, like `protobuf`, `protoc`, and `grpc` |
| 25 | - Next, Netdata should be re-installed from the source. The installer will detect that the required libraries are now available. |
| 26 | configuration: |
| 27 | file: |
| 28 | name: 'exporting.conf' |
| 29 | options: |
| 30 | description: | |
| 31 | The following options can be defined for this exporter. |
| 32 | folding: |
| 33 | title: 'Config options' |
| 34 | enabled: true |
| 35 | list: |
| 36 | - name: 'enabled' |
| 37 | default_value: 'no' |
| 38 | description: 'Enables or disables an exporting connector instance (yes|no).' |
| 39 | required: true |
| 40 | - name: 'destination' |
| 41 | default_value: 'pubsub.googleapis.com' |
| 42 | description: 'Accepts a space separated list of hostnames, IPs (IPv4 and IPv6) and ports to connect to. Netdata will use the first available to send the metrics.' |
| 43 | required: true |
| 44 | detailed_description: | |
| 45 | The format of each item in this list, is: [PROTOCOL:]IP[:PORT]. |
| 46 | - PROTOCOL can be udp or tcp. tcp is the default and only supported by the current exporting engine. |
| 47 | - IP can be XX.XX.XX.XX (IPv4), or [XX:XX...XX:XX] (IPv6). For IPv6 you can to enclose the IP in [] to separate it from the port. |
| 48 | - PORT can be a number of a service name. If omitted, the default port for the exporting connector will be used. |
| 49 | |
| 50 | Example IPv4: |
| 51 | ```yaml |
| 52 | destination = pubsub.googleapis.com |
| 53 | ``` |
| 54 | When multiple servers are defined, Netdata will try the next one when the previous one fails. |
| 55 | - name: 'username' |
| 56 | default_value: 'my_username' |
| 57 | description: 'Username for HTTP authentication' |
| 58 | required: false |
| 59 | - name: 'password' |
| 60 | default_value: 'my_password' |
| 61 | description: 'Password for HTTP authentication' |
| 62 | required: false |
| 63 | - name: 'data source' |
| 64 | default_value: '' |
| 65 | description: 'Selects the kind of data that will be sent to the external database. (as collected|average|sum)' |
| 66 | required: false |
| 67 | - name: 'hostname' |
| 68 | default_value: '[global].hostname' |
| 69 | description: 'The hostname to be used for sending data to the external database server.' |
| 70 | required: false |
| 71 | - name: 'prefix' |
| 72 | default_value: 'Netdata' |
| 73 | description: 'The prefix to add to all metrics.' |
| 74 | required: false |
| 75 | - name: 'update every' |
| 76 | default_value: '10' |
| 77 | description: | |
| 78 | Frequency of sending sending data to the external database, in seconds. |
| 79 | required: false |
| 80 | detailed_description: | |
| 81 | Netdata will add some randomness to this number, to prevent stressing the external server when many Netdata servers |
| 82 | send data to the same database. This randomness does not affect the quality of the data, only the time they are sent. |
| 83 | - name: 'buffer on failures' |
| 84 | default_value: '10' |
| 85 | description: | |
| 86 | The number of iterations (`update every` seconds) to buffer data, when the external database server is not available. |
| 87 | required: false |
| 88 | detailed_description: | |
| 89 | If the server fails to receive the data after that many failures, data loss on the connector instance is expected (Netdata will also log it). |
| 90 | - name: 'timeout ms' |
| 91 | default_value: '2 * update_every * 1000' |
| 92 | description: 'The timeout in milliseconds to wait for the external database server to process the data.' |
| 93 | required: false |
| 94 | - name: 'send hosts matching' |
| 95 | default_value: 'localhost *' |
| 96 | description: | |
| 97 | Hosts filter. Determines which hosts will be sent to the external database. The syntax is [simple patterns](https://github.com/netdata/netdata/tree/master/src/libnetdata/simple_pattern#simple-patterns). |
| 98 | required: false |
| 99 | detailed_description: | |
| 100 | Includes one or more space separated patterns, using * as wildcard (any number of times within each pattern). |
| 101 | The patterns are checked against the hostname (the localhost is always checked as localhost), allowing us to |
| 102 | filter which hosts will be sent to the external database when this Netdata is a central Netdata aggregating multiple hosts. |
| 103 | |
| 104 | A pattern starting with `!` gives a negative match. So to match all hosts named `*db*` except hosts containing `*child*`, |
| 105 | use `!*child* *db*` (so, the order is important: the first pattern matching the hostname will be used - positive or negative). |
| 106 | - name: 'send charts matching' |
| 107 | default_value: '*' |
| 108 | description: | |
| 109 | One or more space separated patterns (use * as wildcard) checked against both chart id and chart name. |
| 110 | required: false |
| 111 | detailed_description: | |
| 112 | A pattern starting with ! gives a negative match. So to match all charts named apps.* except charts ending in *reads, |
| 113 | use !*reads apps.* (so, the order is important: the first pattern matching the chart id or the chart name will be used, |
| 114 | positive or negative). There is also a URL parameter filter that can be used while querying allmetrics. The URL parameter |
| 115 | has a higher priority than the configuration option. |
| 116 | - name: 'send names instead of ids' |
| 117 | default_value: '' |
| 118 | description: 'Controls the metric names Netdata should send to the external database (yes|no).' |
| 119 | required: false |
| 120 | detailed_description: | |
| 121 | Netdata supports names and IDs for charts and dimensions. Usually IDs are unique identifiers as read by the system and names |
| 122 | are human friendly labels (also unique). Most charts and metrics have the same ID and name, but in several cases they are |
| 123 | different : disks with device-mapper, interrupts, QoS classes, statsd synthetic charts, etc. |
| 124 | - name: 'send configured labels' |
| 125 | default_value: '' |
| 126 | description: 'Controls if host labels defined in the `[host labels]` section in `netdata.conf` should be sent to the external database (yes|no).' |
| 127 | required: false |
| 128 | - name: 'send automatic labels' |
| 129 | default_value: '' |
| 130 | description: 'Controls if automatically created labels, like `_os_name` or `_architecture` should be sent to the external database (yes|no).' |
| 131 | required: false |
| 132 | examples: |
| 133 | folding: |
| 134 | enabled: true |
| 135 | title: '' |
| 136 | list: |
| 137 | - name: 'Basic configuration' |
| 138 | folding: |
| 139 | enabled: false |
| 140 | description: | |
| 141 | - Set the destination option to a Pub/Sub service endpoint. pubsub.googleapis.com is the default one. |
| 142 | - Create the credentials JSON file by following Google Cloud's authentication guide. |
| 143 | - The user running the Agent (typically netdata) needs read access to google_cloud_credentials.json, which you can set |
| 144 | `chmod 400 google_cloud_credentials.json; chown netdata google_cloud_credentials.json` |
| 145 | - Set the credentials file option to the full path of the file. |
| 146 | config: | |
| 147 | [pubsub:my_instance] |
| 148 | enabled = yes |
| 149 | destination = pubsub.googleapis.com |
| 150 | credentials file = /etc/netdata/google_cloud_credentials.json |
| 151 | project id = my_project |
| 152 | topic id = my_topic |