| 1 | # netdata python.d.plugin configuration for pandas |
| 2 | # |
| 3 | # This file is in YaML format. Generally the format is: |
| 4 | # |
| 5 | # name: value |
| 6 | # |
| 7 | # There are 2 sections: |
| 8 | # - global variables |
| 9 | # - one or more JOBS |
| 10 | # |
| 11 | # JOBS allow you to collect values from multiple sources. |
| 12 | # Each source will have its own set of charts. |
| 13 | # |
| 14 | # JOB parameters have to be indented (using spaces only, example below). |
| 15 | |
| 16 | # ---------------------------------------------------------------------- |
| 17 | # Global Variables |
| 18 | # These variables set the defaults for all JOBs, however each JOB |
| 19 | # may define its own, overriding the defaults. |
| 20 | |
| 21 | # update_every sets the default data collection frequency. |
| 22 | # If unset, the python.d.plugin default is used. |
| 23 | update_every: 5 |
| 24 | |
| 25 | # priority controls the order of charts at the netdata dashboard. |
| 26 | # Lower numbers move the charts towards the top of the page. |
| 27 | # If unset, the default for python.d.plugin is used. |
| 28 | # priority: 60000 |
| 29 | |
| 30 | # penalty indicates whether to apply penalty to update_every in case of failures. |
| 31 | # Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes. |
| 32 | # penalty: yes |
| 33 | |
| 34 | # autodetection_retry sets the job re-check interval in seconds. |
| 35 | # The job is not deleted if check fails. |
| 36 | # Attempts to start the job are made once every autodetection_retry. |
| 37 | # This feature is disabled by default. |
| 38 | # autodetection_retry: 0 |
| 39 | |
| 40 | # ---------------------------------------------------------------------- |
| 41 | # JOBS (data collection sources) |
| 42 | # |
| 43 | # The default JOBS share the same *name*. JOBS with the same name |
| 44 | # are mutually exclusive. Only one of them will be allowed running at |
| 45 | # any time. This allows autodetection to try several alternatives and |
| 46 | # pick the one that works. |
| 47 | # |
| 48 | # Any number of jobs is supported. |
| 49 | # |
| 50 | # All python.d.plugin JOBS (for all its modules) support a set of |
| 51 | # predefined parameters. These are: |
| 52 | # |
| 53 | # job_name: |
| 54 | # name: myname # the JOB's name as it will appear on the dashboard |
| 55 | # # dashboard (by default is the job_name) |
| 56 | # # JOBs sharing a name are mutually exclusive |
| 57 | # update_every: 1 # the JOB's data collection frequency |
| 58 | # priority: 60000 # the JOB's order on the dashboard |
| 59 | # penalty: yes # the JOB's penalty |
| 60 | # autodetection_retry: 0 # the JOB's re-check interval in seconds |
| 61 | # |
| 62 | # Additionally to the above, example also supports the following: |
| 63 | # |
| 64 | # chart_configs: [<dictionary>] # an array for chart config dictionaries. |
| 65 | # |
| 66 | # ---------------------------------------------------------------------- |
| 67 | # AUTO-DETECTION JOBS |
| 68 | |
| 69 | # Some example configurations, enable this collector, uncomment and example below and restart netdata to enable. |
| 70 | |
| 71 | # example pulling some hourly temperature data, a chart for today forecast (mean,min,max) and another chart for current. |
| 72 | # temperature: |
| 73 | # name: "temperature" |
| 74 | # update_every: 5 |
| 75 | # chart_configs: |
| 76 | # - name: "temperature_forecast_by_city" |
| 77 | # title: "Temperature By City - Today Forecast" |
| 78 | # family: "temperature.today" |
| 79 | # context: "pandas.temperature" |
| 80 | # type: "line" |
| 81 | # units: "Celsius" |
| 82 | # df_steps: > |
| 83 | # pd.DataFrame.from_dict( |
| 84 | # {city: requests.get(f'https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lng}&hourly=temperature_2m').json()['hourly']['temperature_2m'] |
| 85 | # for (city,lat,lng) |
| 86 | # in [ |
| 87 | # ('dublin', 53.3441, -6.2675), |
| 88 | # ('athens', 37.9792, 23.7166), |
| 89 | # ('london', 51.5002, -0.1262), |
| 90 | # ('berlin', 52.5235, 13.4115), |
| 91 | # ('paris', 48.8567, 2.3510), |
| 92 | # ('madrid', 40.4167, -3.7033), |
| 93 | # ('new_york', 40.71, -74.01), |
| 94 | # ('los_angeles', 34.05, -118.24), |
| 95 | # ] |
| 96 | # } |
| 97 | # ); |
| 98 | # df.describe(); # get aggregate stats for each city; |
| 99 | # df.transpose()[['mean', 'max', 'min']].reset_index(); # just take mean, min, max; |
| 100 | # df.rename(columns={'index':'city'}); # some column renaming; |
| 101 | # df.pivot(columns='city').mean().to_frame().reset_index(); # force to be one row per city; |
| 102 | # df.rename(columns={0:'degrees'}); # some column renaming; |
| 103 | # pd.concat([df, df['city']+'_'+df['level_0']], axis=1); # add new column combining city and summary measurement label; |
| 104 | # df.rename(columns={0:'measurement'}); # some column renaming; |
| 105 | # df[['measurement', 'degrees']].set_index('measurement'); # just take two columns we want; |
| 106 | # df.sort_index(); # sort by city name; |
| 107 | # df.transpose(); # transpose so its just one wide row; |
| 108 | # - name: "temperature_current_by_city" |
| 109 | # title: "Temperature By City - Current" |
| 110 | # family: "temperature.current" |
| 111 | # context: "pandas.temperature" |
| 112 | # type: "line" |
| 113 | # units: "Celsius" |
| 114 | # df_steps: > |
| 115 | # pd.DataFrame.from_dict( |
| 116 | # {city: requests.get(f'https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lng}¤t_weather=true').json()['current_weather'] |
| 117 | # for (city,lat,lng) |
| 118 | # in [ |
| 119 | # ('dublin', 53.3441, -6.2675), |
| 120 | # ('athens', 37.9792, 23.7166), |
| 121 | # ('london', 51.5002, -0.1262), |
| 122 | # ('berlin', 52.5235, 13.4115), |
| 123 | # ('paris', 48.8567, 2.3510), |
| 124 | # ('madrid', 40.4167, -3.7033), |
| 125 | # ('new_york', 40.71, -74.01), |
| 126 | # ('los_angeles', 34.05, -118.24), |
| 127 | # ] |
| 128 | # } |
| 129 | # ); |
| 130 | # df.transpose(); |
| 131 | # df[['temperature']]; |
| 132 | # df.transpose(); |
| 133 | |
| 134 | # example showing a read_csv from a url and some light pandas data wrangling. |
| 135 | # pull data in csv format from london demo server and then ratio of user cpus over system cpu averaged over last 60 seconds. |
| 136 | # example_csv: |
| 137 | # name: "example_csv" |
| 138 | # update_every: 2 |
| 139 | # chart_configs: |
| 140 | # - name: "london_system_cpu" |
| 141 | # title: "London System CPU - Ratios" |
| 142 | # family: "london_system_cpu" |
| 143 | # context: "pandas" |
| 144 | # type: "line" |
| 145 | # units: "n" |
| 146 | # df_steps: > |
| 147 | # pd.read_csv('https://london.my-netdata.io/api/v1/data?chart=system.cpu&format=csv&after=-60', storage_options={'User-Agent': 'netdata'}); |
| 148 | # df.drop('time', axis=1); |
| 149 | # df.mean().to_frame().transpose(); |
| 150 | # df.apply(lambda row: (row.user / row.system), axis = 1).to_frame(); |
| 151 | # df.rename(columns={0:'average_user_system_ratio'}); |
| 152 | # df*100; |
| 153 | |
| 154 | # example showing a read_json from a url and some light pandas data wrangling. |
| 155 | # pull data in json format (using requests.get() if json data is too complex for pd.read_json() ) from london demo server and work out 'total_bandwidth'. |
| 156 | # example_json: |
| 157 | # name: "example_json" |
| 158 | # update_every: 2 |
| 159 | # chart_configs: |
| 160 | # - name: "london_system_net" |
| 161 | # title: "London System Net - Total Bandwidth" |
| 162 | # family: "london_system_net" |
| 163 | # context: "pandas" |
| 164 | # type: "area" |
| 165 | # units: "kilobits/s" |
| 166 | # df_steps: > |
| 167 | # pd.DataFrame(requests.get('https://london.my-netdata.io/api/v1/data?chart=system.net&format=json&after=-1').json()['data'], columns=requests.get('https://london.my-netdata.io/api/v1/data?chart=system.net&format=json&after=-1').json()['labels']); |
| 168 | # df.drop('time', axis=1); |
| 169 | # abs(df); |
| 170 | # df.sum(axis=1).to_frame(); |
| 171 | # df.rename(columns={0:'total_bandwidth'}); |
| 172 | |
| 173 | # example showing a read_xml from a url and some light pandas data wrangling. |
| 174 | # pull weather forecast data in xml format, use xpath to pull out temperature forecast. |
| 175 | # example_xml: |
| 176 | # name: "example_xml" |
| 177 | # update_every: 2 |
| 178 | # line_sep: "|" |
| 179 | # chart_configs: |
| 180 | # - name: "temperature_forcast" |
| 181 | # title: "Temperature Forecast" |
| 182 | # family: "temp" |
| 183 | # context: "pandas.temp" |
| 184 | # type: "line" |
| 185 | # units: "celsius" |
| 186 | # df_steps: > |
| 187 | # pd.read_xml('http://metwdb-openaccess.ichec.ie/metno-wdb2ts/locationforecast?lat=54.7210798611;long=-8.7237392806', xpath='./product/time[1]/location/temperature', parser='etree')| |
| 188 | # df.rename(columns={'value': 'dublin'})| |
| 189 | # df[['dublin']]| |
| 190 | |
| 191 | # example showing a read_sql from a postgres database using sqlalchemy. |
| 192 | # note: example assumes a running postgress db on localhost with a netdata users and password netdata. |
| 193 | # sql: |
| 194 | # name: "sql" |
| 195 | # update_every: 5 |
| 196 | # chart_configs: |
| 197 | # - name: "sql" |
| 198 | # title: "SQL Example" |
| 199 | # family: "sql.example" |
| 200 | # context: "example" |
| 201 | # type: "line" |
| 202 | # units: "percent" |
| 203 | # df_steps: > |
| 204 | # pd.read_sql_query( |
| 205 | # sql='\ |
| 206 | # select \ |
| 207 | # random()*100 as metric_1, \ |
| 208 | # random()*100 as metric_2 \ |
| 209 | # ', |
| 210 | # con=create_engine('postgresql://localhost/postgres?user=netdata&password=netdata') |
| 211 | # ); |