@cryptotaxi247 / netdata-1 / commits / b863e5062

Trimmed-median, trimmed-mean and percentile (#13469)

Costa Tsaousis committed Aug 5, 2022 at 12:50 UTC b863e5062ec075eb9b310c758d4efa72a1036b09
19 files changed +1542 -47
CMakeLists.txt
+4
@@ -814,12 +814,16 @@ set(API_PLUGIN_FILES
814 web/api/queries/sum/sum.h
815 web/api/queries/median/median.c
816 web/api/queries/median/median.h
817 + web/api/queries/percentile/percentile.c
818 + web/api/queries/percentile/percentile.h
819 web/api/queries/stddev/stddev.c
820 web/api/queries/stddev/stddev.h
821 web/api/queries/ses/ses.c
822 web/api/queries/ses/ses.h
823 web/api/queries/des/des.c
824 web/api/queries/des/des.h
825 + web/api/queries/trimmed_mean/trimmed_mean.c
826 + web/api/queries/trimmed_mean/trimmed_mean.h
827 web/api/queries/weights.c
828 web/api/queries/weights.h
829 web/api/formatters/rrd2json.c
Makefile.am
+4
@@ -587,6 +587,10 @@ API_PLUGIN_FILES = \
587 web/api/queries/median/median.h \
588 web/api/queries/min/min.c \
589 web/api/queries/min/min.h \
590 + web/api/queries/percentile/percentile.c \
591 + web/api/queries/percentile/percentile.h \
592 + web/api/queries/trimmed_mean/trimmed_mean.c \
593 + web/api/queries/trimmed_mean/trimmed_mean.h \
594 web/api/queries/query.c \
595 web/api/queries/query.h \
596 web/api/queries/rrdr.c \
configure.ac
+2
@@ -1748,9 +1748,11 @@ AC_CONFIG_FILES([
1748 web/api/queries/max/Makefile
1749 web/api/queries/median/Makefile
1750 web/api/queries/min/Makefile
1751 + web/api/queries/percentile/Makefile
1752 web/api/queries/ses/Makefile
1753 web/api/queries/stddev/Makefile
1754 web/api/queries/sum/Makefile
1755 + web/api/queries/trimmed_mean/Makefile
1756 web/api/health/Makefile
1757 web/gui/Makefile
1758 web/gui/dashboard/Makefile
web/api/netdata-swagger.json
+440 -4
@@ -453,7 +453,35 @@
453 "ses",
454 "des",
455 "cv",
456 - "countif"
456 + "countif",
457 + "percentile",
458 + "percentile25",
459 + "percentile50",
460 + "percentile75",
461 + "percentile80",
462 + "percentile90",
463 + "percentile95",
464 + "percentile97",
465 + "percentile98",
466 + "percentile99",
467 + "trimmed-mean",
468 + "trimmed-mean1",
469 + "trimmed-mean2",
470 + "trimmed-mean3",
471 + "trimmed-mean5",
472 + "trimmed-mean10",
473 + "trimmed-mean15",
474 + "trimmed-mean20",
475 + "trimmed-mean25",
476 + "trimmed-median",
477 + "trimmed-median1",
478 + "trimmed-median2",
479 + "trimmed-median3",
480 + "trimmed-median5",
481 + "trimmed-median10",
482 + "trimmed-median15",
483 + "trimmed-median20",
484 + "trimmed-median25"
485 ],
486 "default": "average"
487 }
@@ -690,7 +718,39 @@
718 "median",
719 "stddev",
720 "sum",
693 - "incremental-sum"
721 + "incremental-sum",
722 + "ses",
723 + "des",
724 + "cv",
725 + "countif",
726 + "percentile",
727 + "percentile25",
728 + "percentile50",
729 + "percentile75",
730 + "percentile80",
731 + "percentile90",
732 + "percentile95",
733 + "percentile97",
734 + "percentile98",
735 + "percentile99",
736 + "trimmed-mean",
737 + "trimmed-mean1",
738 + "trimmed-mean2",
739 + "trimmed-mean3",
740 + "trimmed-mean5",
741 + "trimmed-mean10",
742 + "trimmed-mean15",
743 + "trimmed-mean20",
744 + "trimmed-mean25",
745 + "trimmed-median",
746 + "trimmed-median1",
747 + "trimmed-median2",
748 + "trimmed-median3",
749 + "trimmed-median5",
750 + "trimmed-median10",
751 + "trimmed-median15",
752 + "trimmed-median20",
753 + "trimmed-median25"
754 ],
755 "default": "average"
756 }
@@ -1362,7 +1422,7 @@
1422 "/metric_correlations": {
1423 "get": {
1424 "summary": "Analyze all the metrics to find their correlations",
1365 - "description": "Given two time-windows (baseline, highlight), it goes through all the available metrics, querying both windows and tries to find how these two windows relate to each other. It supports multiple algorithms to do so. The result is a list of all metrics evaluated, weighted for 0.0 (the two windows are more different) to 1.0 (the two windows are similar). The algorithm adjusts automatically the baseline window to be a power of two multiple of the highlighted (1, 2, 4, 8, etc).",
1425 + "description": "THIS ENDPOINT IS OBSOLETE. Use the /weights endpoint. Given two time-windows (baseline, highlight), it goes through all the available metrics, querying both windows and tries to find how these two windows relate to each other. It supports multiple algorithms to do so. The result is a list of all metrics evaluated, weighted for 0.0 (the two windows are more different) to 1.0 (the two windows are similar). The algorithm adjusts automatically the baseline window to be a power of two multiple of the highlighted (1, 2, 4, 8, etc).",
1426 "parameters": [
1427 {
1428 "name": "baseline_after",
@@ -1499,7 +1559,35 @@
1559 "ses",
1560 "des",
1561 "cv",
1502 - "countif"
1562 + "countif",
1563 + "percentile",
1564 + "percentile25",
1565 + "percentile50",
1566 + "percentile75",
1567 + "percentile80",
1568 + "percentile90",
1569 + "percentile95",
1570 + "percentile97",
1571 + "percentile98",
1572 + "percentile99",
1573 + "trimmed-mean",
1574 + "trimmed-mean1",
1575 + "trimmed-mean2",
1576 + "trimmed-mean3",
1577 + "trimmed-mean5",
1578 + "trimmed-mean10",
1579 + "trimmed-mean15",
1580 + "trimmed-mean20",
1581 + "trimmed-mean25",
1582 + "trimmed-median",
1583 + "trimmed-median1",
1584 + "trimmed-median2",
1585 + "trimmed-median3",
1586 + "trimmed-median5",
1587 + "trimmed-median10",
1588 + "trimmed-median15",
1589 + "trimmed-median20",
1590 + "trimmed-median25"
1591 ],
1592 "default": "average"
1593 }
@@ -1540,6 +1628,236 @@
1628 }
1629 }
1630 }
1631 + },
1632 + "/weights": {
1633 + "get": {
1634 + "summary": "Analyze all the metrics using an algorithm and score them accordingly",
1635 + "description": "This endpoint goes through all metrics and scores them according to an algorithm.",
1636 + "parameters": [
1637 + {
1638 + "name": "baseline_after",
1639 + "in": "query",
1640 + "description": "This parameter can either be an absolute timestamp specifying the starting point of baseline window, or a relative number of seconds (negative, relative to parameter baseline_before). Netdata will assume it is a relative number if it is less that 3 years (in seconds). This parameter is used in KS2 and VOLUME algorithms.",
1641 + "required": false,
1642 + "allowEmptyValue": false,
1643 + "schema": {
1644 + "type": "number",
1645 + "format": "integer",
1646 + "default": -300
1647 + }
1648 + },
1649 + {
1650 + "name": "baseline_before",
1651 + "in": "query",
1652 + "description": "This parameter can either be an absolute timestamp specifying the ending point of the baseline window, or a relative number of seconds (negative), relative to the last collected timestamp. Netdata will assume it is a relative number if it is less than 3 years (in seconds). This parameter is used in KS2 and VOLUME algorithms.",
1653 + "required": false,
1654 + "schema": {
1655 + "type": "number",
1656 + "format": "integer",
1657 + "default": -60
1658 + }
1659 + },
1660 + {
1661 + "name": "after",
1662 + "in": "query",
1663 + "description": "This parameter can either be an absolute timestamp specifying the starting point of highlighted window, or a relative number of seconds (negative, relative to parameter highlight_before). Netdata will assume it is a relative number if it is less that 3 years (in seconds).",
1664 + "required": false,
1665 + "allowEmptyValue": false,
1666 + "schema": {
1667 + "type": "number",
1668 + "format": "integer",
1669 + "default": -60
1670 + }
1671 + },
1672 + {
1673 + "name": "before",
1674 + "in": "query",
1675 + "description": "This parameter can either be an absolute timestamp specifying the ending point of the highlighted window, or a relative number of seconds (negative), relative to the last collected timestamp. Netdata will assume it is a relative number if it is less than 3 years (in seconds).",
1676 + "required": false,
1677 + "schema": {
1678 + "type": "number",
1679 + "format": "integer",
1680 + "default": 0
1681 + }
1682 + },
1683 + {
1684 + "name": "context",
1685 + "in": "query",
1686 + "description": "A simple pattern matching the contexts to evaluate.",
1687 + "required": false,
1688 + "allowEmptyValue": false,
1689 + "schema": {
1690 + "type": "string"
1691 + }
1692 + },
1693 + {
1694 + "name": "points",
1695 + "in": "query",
1696 + "description": "The number of points to be evaluated for the highlighted window. The baseline window will be adjusted automatically to receive a proportional amount of points. This parameter is only used by the KS2 algorithm.",
1697 + "required": false,
1698 + "allowEmptyValue": false,
1699 + "schema": {
1700 + "type": "number",
1701 + "format": "integer",
1702 + "default": 500
1703 + }
1704 + },
1705 + {
1706 + "name": "method",
1707 + "in": "query",
1708 + "description": "the algorithm to run",
1709 + "required": false,
1710 + "schema": {
1711 + "type": "string",
1712 + "enum": [
1713 + "ks2",
1714 + "volume",
1715 + "anomaly-rate"
1716 + ],
1717 + "default": "anomaly-rate"
1718 + }
1719 + },
1720 + {
1721 + "name": "tier",
1722 + "in": "query",
1723 + "description": "Use the specified database tier",
1724 + "required": false,
1725 + "allowEmptyValue": false,
1726 + "schema": {
1727 + "type": "number",
1728 + "format": "integer"
1729 + }
1730 + },
1731 + {
1732 + "name": "timeout",
1733 + "in": "query",
1734 + "description": "Cancel the query if to takes more that this amount of milliseconds.",
1735 + "required": false,
1736 + "allowEmptyValue": false,
1737 + "schema": {
1738 + "type": "number",
1739 + "format": "integer",
1740 + "default": 60000
1741 + }
1742 + },
1743 + {
1744 + "name": "options",
1745 + "in": "query",
1746 + "description": "Options that affect data generation.",
1747 + "required": false,
1748 + "allowEmptyValue": false,
1749 + "schema": {
1750 + "type": "array",
1751 + "items": {
1752 + "type": "string",
1753 + "enum": [
1754 + "min2max",
1755 + "abs",
1756 + "absolute",
1757 + "absolute-sum",
1758 + "null2zero",
1759 + "percentage",
1760 + "unaligned",
1761 + "nonzero",
1762 + "anomaly-bit",
1763 + "raw"
1764 + ]
1765 + },
1766 + "default": [
1767 + "null2zero",
1768 + "nonzero",
1769 + "unaligned"
1770 + ]
1771 + }
1772 + },
1773 + {
1774 + "name": "group",
1775 + "in": "query",
1776 + "description": "The grouping method. If multiple collected values are to be grouped in order to return fewer points, this parameters defines the method of grouping. methods supported \"min\", \"max\", \"average\", \"sum\", \"incremental-sum\". \"max\" is actually calculated on the absolute value collected (so it works for both positive and negative dimensions to return the most extreme value in either direction).",
1777 + "required": true,
1778 + "allowEmptyValue": false,
1779 + "schema": {
1780 + "type": "string",
1781 + "enum": [
1782 + "min",
1783 + "max",
1784 + "average",
1785 + "median",
1786 + "stddev",
1787 + "sum",
1788 + "incremental-sum",
1789 + "ses",
1790 + "des",
1791 + "cv",
1792 + "countif",
1793 + "percentile",
1794 + "percentile25",
1795 + "percentile50",
1796 + "percentile75",
1797 + "percentile80",
1798 + "percentile90",
1799 + "percentile95",
1800 + "percentile97",
1801 + "percentile98",
1802 + "percentile99",
1803 + "trimmed-mean",
1804 + "trimmed-mean1",
1805 + "trimmed-mean2",
1806 + "trimmed-mean3",
1807 + "trimmed-mean5",
1808 + "trimmed-mean10",
1809 + "trimmed-mean15",
1810 + "trimmed-mean20",
1811 + "trimmed-mean25",
1812 + "trimmed-median",
1813 + "trimmed-median1",
1814 + "trimmed-median2",
1815 + "trimmed-median3",
1816 + "trimmed-median5",
1817 + "trimmed-median10",
1818 + "trimmed-median15",
1819 + "trimmed-median20",
1820 + "trimmed-median25"
1821 + ],
1822 + "default": "average"
1823 + }
1824 + },
1825 + {
1826 + "name": "group_options",
1827 + "in": "query",
1828 + "description": "When the group function supports additional parameters, this field can be used to pass them to it. Currently only \"countif\" supports this.",
1829 + "required": false,
1830 + "allowEmptyValue": false,
1831 + "schema": {
1832 + "type": "string"
1833 + }
1834 + }
1835 + ],
1836 + "responses": {
1837 + "200": {
1838 + "description": "JSON object with weights for each context, chart and dimension.",
1839 + "content": {
1840 + "application/json": {
1841 + "schema": {
1842 + "$ref": "#/components/schemas/weights"
1843 + }
1844 + }
1845 + }
1846 + },
1847 + "400": {
1848 + "description": "The given parameters are invalid."
1849 + },
1850 + "403": {
1851 + "description": "metrics correlations are not enabled on this Netdata Agent."
1852 + },
1853 + "404": {
1854 + "description": "No charts could be found, or the method that correlated the metrics did not produce any result."
1855 + },
1856 + "504": {
1857 + "description": "Timeout - the query took too long and has been cancelled."
1858 + }
1859 + }
1860 + }
1861 }
1862 },
1863 "servers": [
@@ -2785,6 +3103,124 @@
3103 }
3104 }
3105 }
3106 + },
3107 + "weights": {
3108 + "type": "object",
3109 + "properties": {
3110 + "after": {
3111 + "description": "the start time of the highlighted window",
3112 + "type": "integer"
3113 + },
3114 + "before": {
3115 + "description": "the end time of the highlighted window",
3116 + "type": "integer"
3117 + },
3118 + "duration": {
3119 + "description": "the duration of the highlighted window",
3120 + "type": "integer"
3121 + },
3122 + "points": {
3123 + "description": "the points of the highlighted window",
3124 + "type": "integer"
3125 + },
3126 + "baseline_after": {
3127 + "description": "the start time of the baseline window",
3128 + "type": "integer"
3129 + },
3130 + "baseline_before": {
3131 + "description": "the end time of the baseline window",
3132 + "type": "integer"
3133 + },
3134 + "baseline_duration": {
3135 + "description": "the duration of the baseline window",
3136 + "type": "integer"
3137 + },
3138 + "baseline_points": {
3139 + "description": "the points of the baseline window",
3140 + "type": "integer"
3141 + },
3142 + "group": {
3143 + "description": "the grouping method across time",
3144 + "type": "string"
3145 + },
3146 + "method": {
3147 + "description": "the correlation method used",
3148 + "type": "string"
3149 + },
3150 + "options": {
3151 + "description": "a comma separated list of the query options set",
3152 + "type": "string"
3153 + },
3154 + "correlated_dimensions": {
3155 + "description": "the number of dimensions returned in the result"
3156 + },
3157 + "total_dimensions_count": {
3158 + "description": "the total number of dimensions evaluated",
3159 + "type": "integer"
3160 + },
3161 + "statistics": {
3162 + "type": "object",
3163 + "properties": {
3164 + "query_time_ms": {
3165 + "type": "number"
3166 + },
3167 + "db_queries": {
3168 + "type": "integer"
3169 + },
3170 + "db_points_read": {
3171 + "type": "integer"
3172 + },
3173 + "query_result_points": {
3174 + "type": "integer"
3175 + },
3176 + "binary_searches": {
3177 + "type": "integer"
3178 + }
3179 + }
3180 + },
3181 + "contexts": {
3182 + "description": "A dictionary of weighted context objects.",
3183 + "type": "object",
3184 + "additionalProperties": {
3185 + "$ref": "#/components/schemas/weighted_context"
3186 + }
3187 + }
3188 + }
3189 + },
3190 + "weighted_context": {
3191 + "type": "object",
3192 + "properties": {
3193 + "weight": {
3194 + "description": "The average weight of the context.",
3195 + "type": "number"
3196 + },
3197 + "charts": {
3198 + "description": "A dictionary of weighted chart objects.",
3199 + "type": "object",
3200 + "additionalProperties": {
3201 + "$ref": "#/components/schemas/weighted_chart"
3202 + }
3203 + }
3204 + }
3205 + },
3206 + "weighted_chart": {
3207 + "type": "object",
3208 + "properties": {
3209 + "weight": {
3210 + "description": "The average weight of the context.",
3211 + "type": "number"
3212 + },
3213 + "dimensions": {
3214 + "description": "A dictionary of weighted dimensions.",
3215 + "type": "object",
3216 + "additionalProperties": {
3217 + "$ref": "#/components/schemas/weighted_dimension"
3218 + }
3219 + }
3220 + }
3221 + },
3222 + "weighted_dimension": {
3223 + "type": "number"
3224 }
3225 }
3226 }
web/api/netdata-swagger.yaml
+145 -24
@@ -365,6 +365,34 @@ paths:
365 - des
366 - cv
367 - countif
368 + - percentile
369 + - percentile25
370 + - percentile50
371 + - percentile75
372 + - percentile80
373 + - percentile90
374 + - percentile95
375 + - percentile97
376 + - percentile98
377 + - percentile99
378 + - trimmed-mean
379 + - trimmed-mean1
380 + - trimmed-mean2
381 + - trimmed-mean3
382 + - trimmed-mean5
383 + - trimmed-mean10
384 + - trimmed-mean15
385 + - trimmed-mean20
386 + - trimmed-mean25
387 + - trimmed-median
388 + - trimmed-median1
389 + - trimmed-median2
390 + - trimmed-median3
391 + - trimmed-median5
392 + - trimmed-median10
393 + - trimmed-median15
394 + - trimmed-median20
395 + - trimmed-median25
396 default: average
397 - name: group_options
398 in: query
@@ -575,6 +603,38 @@ paths:
603 - stddev
604 - sum
605 - incremental-sum
606 + - ses
607 + - des
608 + - cv
609 + - countif
610 + - percentile
611 + - percentile25
612 + - percentile50
613 + - percentile75
614 + - percentile80
615 + - percentile90
616 + - percentile95
617 + - percentile97
618 + - percentile98
619 + - percentile99
620 + - trimmed-mean
621 + - trimmed-mean1
622 + - trimmed-mean2
623 + - trimmed-mean3
624 + - trimmed-mean5
625 + - trimmed-mean10
626 + - trimmed-mean15
627 + - trimmed-mean20
628 + - trimmed-mean25
629 + - trimmed-median
630 + - trimmed-median1
631 + - trimmed-median2
632 + - trimmed-median3
633 + - trimmed-median5
634 + - trimmed-median10
635 + - trimmed-median15
636 + - trimmed-median20
637 + - trimmed-median25
638 default: average
639 - name: options
640 in: query
@@ -1238,6 +1298,34 @@ paths:
1298 - des
1299 - cv
1300 - countif
1301 + - percentile
1302 + - percentile25
1303 + - percentile50
1304 + - percentile75
1305 + - percentile80
1306 + - percentile90
1307 + - percentile95
1308 + - percentile97
1309 + - percentile98
1310 + - percentile99
1311 + - trimmed-mean
1312 + - trimmed-mean1
1313 + - trimmed-mean2
1314 + - trimmed-mean3
1315 + - trimmed-mean5
1316 + - trimmed-mean10
1317 + - trimmed-mean15
1318 + - trimmed-mean20
1319 + - trimmed-mean25
1320 + - trimmed-median
1321 + - trimmed-median1
1322 + - trimmed-median2
1323 + - trimmed-median3
1324 + - trimmed-median5
1325 + - trimmed-median10
1326 + - trimmed-median15
1327 + - trimmed-median20
1328 + - trimmed-median25
1329 default: average
1330 - name: group_options
1331 in: query
@@ -1413,6 +1501,34 @@ paths:
1501 - des
1502 - cv
1503 - countif
1504 + - percentile
1505 + - percentile25
1506 + - percentile50
1507 + - percentile75
1508 + - percentile80
1509 + - percentile90
1510 + - percentile95
1511 + - percentile97
1512 + - percentile98
1513 + - percentile99
1514 + - trimmed-mean
1515 + - trimmed-mean1
1516 + - trimmed-mean2
1517 + - trimmed-mean3
1518 + - trimmed-mean5
1519 + - trimmed-mean10
1520 + - trimmed-mean15
1521 + - trimmed-mean20
1522 + - trimmed-mean25
1523 + - trimmed-median
1524 + - trimmed-median1
1525 + - trimmed-median2
1526 + - trimmed-median3
1527 + - trimmed-median5
1528 + - trimmed-median10
1529 + - trimmed-median15
1530 + - trimmed-median20
1531 + - trimmed-median25
1532 default: average
1533 - name: group_options
1534 in: query
@@ -1428,7 +1544,7 @@ paths:
1544 content:
1545 application/json:
1546 schema:
1431 - $ref: "#/components/schemas/weight"
1547 + $ref: "#/components/schemas/weights"
1548 "400":
1549 description: The given parameters are invalid.
1550 "403":
@@ -2373,7 +2489,7 @@ components:
2489 type: number
2490 dimension2-name:
2491 type: number
2376 - weight:
2492 + weights:
2493 type: object
2494 properties:
2495 after:
@@ -2428,26 +2544,31 @@ components:
2544 binary_searches:
2545 type: integer
2546 contexts:
2547 + description: A dictionary of weighted context objects.
2548 type: object
2432 - description: An object containing context objects.
2433 - properties:
2434 - contextX:
2435 - type: object
2436 - properties:
2437 - charts:
2438 - type: object
2439 - properties:
2440 - chartX:
2441 - type: object
2442 - properties:
2443 - dimensions:
2444 - type: object
2445 - properties:
2446 - dimensionX:
2447 - type: number
2448 - weight:
2449 - description: The average chart weight
2450 - type: number
2451 - weight:
2452 - description: The average context weight
2453 - type: number
2549 + additionalProperties:
2550 + $ref: '#/components/schemas/weighted_context'
2551 + weighted_context:
2552 + type: object
2553 + properties:
2554 + weight:
2555 + description: The average weight of the context.
2556 + type: number
2557 + charts:
2558 + description: A dictionary of weighted chart objects.
2559 + type: object
2560 + additionalProperties:
2561 + $ref: '#/components/schemas/weighted_chart'
2562 + weighted_chart:
2563 + type: object
2564 + properties:
2565 + weight:
2566 + description: The average weight of the context.
2567 + type: number
2568 + dimensions:
2569 + description: A dictionary of weighted dimensions.
2570 + type: object
2571 + additionalProperties:
2572 + $ref: '#/components/schemas/weighted_dimension'
2573 + weighted_dimension:
2574 + type: number
web/api/queries/Makefile.am
+2
@@ -12,8 +12,10 @@ SUBDIRS = \
12 min \
13 sum \
14 median \
15 + percentile \
16 ses \
17 stddev \
18 + trimmed_mean \
19 $(NULL)
20
21 dist_noinst_DATA = \
web/api/queries/median/README.md
+16 -1
@@ -13,6 +13,20 @@ The median is the value separating the higher half from the lower half of a data
13 `median` is not an accurate average. However, it eliminates all spikes, by sorting
14 all the values in a period, and selecting the value in the middle of the sorted array.
15
16 +Netdata also supports `trimmed-median`, which trims a percentage of the smaller and bigger values prior to finding the
17 +median. The following `trimmed-median` functions are defined:
18 +
19 +- `trimmed-median1`
20 +- `trimmed-median2`
21 +- `trimmed-median3`
22 +- `trimmed-median5`
23 +- `trimmed-median10`
24 +- `trimmed-median15`
25 +- `trimmed-median20`
26 +- `trimmed-median25`
27 +
28 +The function `trimmed-median` is an alias for `trimmed-median5`.
29 +
30 ## how to use
31
32 Use it in alarms like this:
@@ -27,7 +41,8 @@ lookup: median -1m unaligned of my_dimension
41 `median` does not change the units. For example, if the chart units is `requests/sec`, the result
42 will be again expressed in the same units.
43
30 -It can also be used in APIs and badges as `&group=median` in the URL.
44 +It can also be used in APIs and badges as `&group=median` in the URL. Additionally, a percentage may be given with
45 +`&group_options=` to trim all small and big values before finding the median.
46
47 ## Examples
48
web/api/queries/median/median.c
+75 -17
@@ -2,28 +2,65 @@
2
3 #include "median.h"
4
5 -
5 // ----------------------------------------------------------------------------
6 // median
7
8 struct grouping_median {
9 size_t series_size;
10 size_t next_pos;
11 + NETDATA_DOUBLE percent;
12
13 NETDATA_DOUBLE *series;
14 };
15
16 -void grouping_create_median(RRDR *r, const char *options __maybe_unused) {
16 +void grouping_create_median_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
17 long entries = r->group;
18 - if(entries < 0) entries = 0;
18 + if(entries < 10) entries = 10;
19
20 struct grouping_median *g = (struct grouping_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_median));
21 g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
22 g->series_size = (size_t)entries;
23
24 + g->percent = def;
25 + if(options && *options) {
26 + g->percent = str2ndd(options, NULL);
27 + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
28 + if(g->percent < 0.0) g->percent = 0.0;
29 + if(g->percent > 50.0) g->percent = 50.0;
30 + }
31 +
32 + g->percent = g->percent / 100.0;
33 r->internal.grouping_data = g;
34 }
35
36 +void grouping_create_median(RRDR *r, const char *options) {
37 + grouping_create_median_internal(r, options, 0.0);
38 +}
39 +void grouping_create_trimmed_median1(RRDR *r, const char *options) {
40 + grouping_create_median_internal(r, options, 1.0);
41 +}
42 +void grouping_create_trimmed_median2(RRDR *r, const char *options) {
43 + grouping_create_median_internal(r, options, 2.0);
44 +}
45 +void grouping_create_trimmed_median3(RRDR *r, const char *options) {
46 + grouping_create_median_internal(r, options, 3.0);
47 +}
48 +void grouping_create_trimmed_median5(RRDR *r, const char *options) {
49 + grouping_create_median_internal(r, options, 5.0);
50 +}
51 +void grouping_create_trimmed_median10(RRDR *r, const char *options) {
52 + grouping_create_median_internal(r, options, 10.0);
53 +}
54 +void grouping_create_trimmed_median15(RRDR *r, const char *options) {
55 + grouping_create_median_internal(r, options, 15.0);
56 +}
57 +void grouping_create_trimmed_median20(RRDR *r, const char *options) {
58 + grouping_create_median_internal(r, options, 20.0);
59 +}
60 +void grouping_create_trimmed_median25(RRDR *r, const char *options) {
61 + grouping_create_median_internal(r, options, 25.0);
62 +}
63 +
64 // resets when switches dimensions
65 // so, clear everything to restart
66 void grouping_reset_median(RRDR *r) {
@@ -46,37 +83,58 @@ void grouping_add_median(RRDR *r, NETDATA_DOUBLE value) {
83 g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
84 g->series_size *= 2;
85 }
49 - else
50 - g->series[g->next_pos++] = (NETDATA_DOUBLE)value;
86 +
87 + g->series[g->next_pos++] = value;
88 }
89
90 NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
91 struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data;
92
93 + size_t available_slots = g->next_pos;
94 NETDATA_DOUBLE value;
95
58 - if(unlikely(!g->next_pos)) {
96 + if(unlikely(!available_slots)) {
97 value = 0.0;
98 *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
99 }
100 + else if(available_slots == 1) {
101 + value = g->series[0];
102 + }
103 else {
63 - if(g->next_pos > 1) {
64 - sort_series(g->series, g->next_pos);
65 - value = (NETDATA_DOUBLE)median_on_sorted_series(g->series, g->next_pos);
66 - }
67 - else
68 - value = (NETDATA_DOUBLE)g->series[0];
104 + sort_series(g->series, available_slots);
105 +
106 + size_t start_slot = 0;
107 + size_t end_slot = available_slots - 1;
108 +
109 + if(g->percent > 0.0) {
110 + NETDATA_DOUBLE min = g->series[0];
111 + NETDATA_DOUBLE max = g->series[available_slots - 1];
112 + NETDATA_DOUBLE delta = (max - min) * g->percent;
113 +
114 + NETDATA_DOUBLE wanted_min = min + delta;
115 + NETDATA_DOUBLE wanted_max = max - delta;
116 +
117 + for (start_slot = 0; start_slot < available_slots; start_slot++)
118 + if (g->series[start_slot] >= wanted_min) break;
119
70 - if(!netdata_double_isnumber(value)) {
71 - value = 0.0;
72 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
120 + for (end_slot = available_slots - 1; end_slot > start_slot; end_slot--)
121 + if (g->series[end_slot] <= wanted_max) break;
122 }
123
75 - //log_series_to_stderr(g->series, g->next_pos, value, "median");
124 + if(start_slot == end_slot)
125 + value = g->series[start_slot];
126 + else
127 + value = median_on_sorted_series(&g->series[start_slot], end_slot - start_slot + 1);
128 }
129
130 + if(unlikely(!netdata_double_isnumber(value))) {
131 + value = 0.0;
132 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
133 + }
134 +
135 + //log_series_to_stderr(g->series, g->next_pos, value, "median");
136 +
137 g->next_pos = 0;
138
139 return value;
140 }
82 -
web/api/queries/median/median.h
+9 -1
@@ -6,7 +6,15 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -extern void grouping_create_median(RRDR *r, const char *options __maybe_unused);
9 +extern void grouping_create_median(RRDR *r, const char *options);
10 +extern void grouping_create_trimmed_median1(RRDR *r, const char *options);
11 +extern void grouping_create_trimmed_median2(RRDR *r, const char *options);
12 +extern void grouping_create_trimmed_median3(RRDR *r, const char *options);
13 +extern void grouping_create_trimmed_median5(RRDR *r, const char *options);
14 +extern void grouping_create_trimmed_median10(RRDR *r, const char *options);
15 +extern void grouping_create_trimmed_median15(RRDR *r, const char *options);
16 +extern void grouping_create_trimmed_median20(RRDR *r, const char *options);
17 +extern void grouping_create_trimmed_median25(RRDR *r, const char *options);
18 extern void grouping_reset_median(RRDR *r);
19 extern void grouping_free_median(RRDR *r);
20 extern void grouping_add_median(RRDR *r, NETDATA_DOUBLE value);
web/api/queries/percentile/Makefile.am new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +dist_noinst_DATA = \
7 + README.md \
8 + $(NULL)
web/api/queries/percentile/README.md new
+58
@@ -0,0 +1,58 @@
1 +<!--
2 +title: "Percentile"
3 +description: "Use percentile in API queries and health entities to find the 'percentile' value from a sample, eliminating any unwanted spikes in the returned metrics."
4 +custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/percentile/README.md
5 +-->
6 +
7 +# Percentile
8 +
9 +The percentile is the average value of a series using only the smaller N percentile of the values.
10 +(a population or a probability distribution).
11 +
12 +Netdata applies linear interpolation on the last point, if the percentile requested does not give a round number of
13 +points.
14 +
15 +The following percentile aliases are defined:
16 +
17 +- `percentile25`
18 +- `percentile50`
19 +- `percentile75`
20 +- `percentile80`
21 +- `percentile90`
22 +- `percentile95`
23 +- `percentile97`
24 +- `percentile98`
25 +- `percentile99`
26 +
27 +The default `percentile` is an alias for `percentile95`.
28 +Any percentile may be requested using the `group_options` query parameter.
29 +
30 +## how to use
31 +
32 +Use it in alarms like this:
33 +
34 +```
35 + alarm: my_alarm
36 + on: my_chart
37 +lookup: percentile95 -1m unaligned of my_dimension
38 + warn: $this > 1000
39 +```
40 +
41 +`percentile` does not change the units. For example, if the chart units is `requests/sec`, the result
42 +will be again expressed in the same units.
43 +
44 +It can also be used in APIs and badges as `&group=percentile` in the URL and the additional parameter `group_options`
45 +may be used to request any percentile (e.g. `&group=percentile&group_options=96`).
46 +
47 +## Examples
48 +
49 +Examining last 1 minute `successful` web server responses:
50 +
51 +- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=min&after=-60&label=min)
52 +- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=average&after=-60&label=average)
53 +- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=percentile95&after=-60&label=percentile95&value_color=orange)
54 +- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=max&after=-60&label=max)
55 +
56 +## References
57 +
58 +- <https://en.wikipedia.org/wiki/Percentile>.
web/api/queries/percentile/percentile.c new
+169
@@ -0,0 +1,169 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "percentile.h"
4 +
5 +// ----------------------------------------------------------------------------
6 +// median
7 +
8 +struct grouping_percentile {
9 + size_t series_size;
10 + size_t next_pos;
11 + NETDATA_DOUBLE percent;
12 +
13 + NETDATA_DOUBLE *series;
14 +};
15 +
16 +static void grouping_create_percentile_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
17 + long entries = r->group;
18 + if(entries < 10) entries = 10;
19 +
20 + struct grouping_percentile *g = (struct grouping_percentile *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_percentile));
21 + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
22 + g->series_size = (size_t)entries;
23 +
24 + g->percent = def;
25 + if(options && *options) {
26 + g->percent = str2ndd(options, NULL);
27 + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
28 + if(g->percent < 0.0) g->percent = 0.0;
29 + if(g->percent > 100.0) g->percent = 100.0;
30 + }
31 +
32 + g->percent = g->percent / 100.0;
33 + r->internal.grouping_data = g;
34 +}
35 +
36 +void grouping_create_percentile25(RRDR *r, const char *options) {
37 + grouping_create_percentile_internal(r, options, 25.0);
38 +}
39 +void grouping_create_percentile50(RRDR *r, const char *options) {
40 + grouping_create_percentile_internal(r, options, 50.0);
41 +}
42 +void grouping_create_percentile75(RRDR *r, const char *options) {
43 + grouping_create_percentile_internal(r, options, 75.0);
44 +}
45 +void grouping_create_percentile80(RRDR *r, const char *options) {
46 + grouping_create_percentile_internal(r, options, 80.0);
47 +}
48 +void grouping_create_percentile90(RRDR *r, const char *options) {
49 + grouping_create_percentile_internal(r, options, 90.0);
50 +}
51 +void grouping_create_percentile95(RRDR *r, const char *options) {
52 + grouping_create_percentile_internal(r, options, 95.0);
53 +}
54 +void grouping_create_percentile97(RRDR *r, const char *options) {
55 + grouping_create_percentile_internal(r, options, 97.0);
56 +}
57 +void grouping_create_percentile98(RRDR *r, const char *options) {
58 + grouping_create_percentile_internal(r, options, 98.0);
59 +}
60 +void grouping_create_percentile99(RRDR *r, const char *options) {
61 + grouping_create_percentile_internal(r, options, 99.0);
62 +}
63 +
64 +// resets when switches dimensions
65 +// so, clear everything to restart
66 +void grouping_reset_percentile(RRDR *r) {
67 + struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
68 + g->next_pos = 0;
69 +}
70 +
71 +void grouping_free_percentile(RRDR *r) {
72 + struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
73 + if(g) onewayalloc_freez(r->internal.owa, g->series);
74 +
75 + onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
76 + r->internal.grouping_data = NULL;
77 +}
78 +
79 +void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value) {
80 + struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
81 +
82 + if(unlikely(g->next_pos >= g->series_size)) {
83 + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
84 + g->series_size *= 2;
85 + }
86 +
87 + g->series[g->next_pos++] = value;
88 +}
89 +
90 +NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
91 + struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
92 +
93 + NETDATA_DOUBLE value;
94 + size_t available_slots = g->next_pos;
95 +
96 + if(unlikely(!available_slots)) {
97 + value = 0.0;
98 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
99 + }
100 + else if(available_slots == 1) {
101 + value = g->series[0];
102 + }
103 + else {
104 + sort_series(g->series, available_slots);
105 +
106 + NETDATA_DOUBLE min = g->series[0];
107 + NETDATA_DOUBLE max = g->series[available_slots - 1];
108 +
109 + if (min != max) {
110 + size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent);
111 + if(!slots_to_use) slots_to_use = 1;
112 +
113 + NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots;
114 + NETDATA_DOUBLE percent_delta = g->percent - percent_to_use;
115 +
116 + NETDATA_DOUBLE percent_interpolation_slot = 0.0;
117 + NETDATA_DOUBLE percent_last_slot = 0.0;
118 + if(percent_delta > 0.0) {
119 + NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots;
120 + NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use;
121 +
122 + percent_interpolation_slot = percent_delta / percent_1slot;
123 + percent_last_slot = 1 - percent_interpolation_slot;
124 + }
125 +
126 + int start_slot, stop_slot, step, last_slot, interpolation_slot;
127 + if(min >= 0.0 && max >= 0.0) {
128 + start_slot = 0;
129 + stop_slot = start_slot + (int)slots_to_use;
130 + last_slot = stop_slot - 1;
131 + interpolation_slot = stop_slot;
132 + step = 1;
133 + }
134 + else {
135 + start_slot = (int)available_slots - 1;
136 + stop_slot = start_slot - (int)slots_to_use;
137 + last_slot = stop_slot + 1;
138 + interpolation_slot = stop_slot;
139 + step = -1;
140 + }
141 +
142 + value = 0.0;
143 + for(int slot = start_slot; slot != stop_slot ; slot += step)
144 + value += g->series[slot];
145 +
146 + size_t counted = slots_to_use;
147 + if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) {
148 + value += g->series[interpolation_slot] * percent_interpolation_slot;
149 + value += g->series[last_slot] * percent_last_slot;
150 + counted++;
151 + }
152 +
153 + value = value / (NETDATA_DOUBLE)counted;
154 + }
155 + else
156 + value = min;
157 + }
158 +
159 + if(unlikely(!netdata_double_isnumber(value))) {
160 + value = 0.0;
161 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
162 + }
163 +
164 + //log_series_to_stderr(g->series, g->next_pos, value, "percentile");
165 +
166 + g->next_pos = 0;
167 +
168 + return value;
169 +}
web/api/queries/percentile/percentile.h new
+23
@@ -0,0 +1,23 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_API_QUERIES_PERCENTILE_H
4 +#define NETDATA_API_QUERIES_PERCENTILE_H
5 +
6 +#include "../query.h"
7 +#include "../rrdr.h"
8 +
9 +extern void grouping_create_percentile25(RRDR *r, const char *options);
10 +extern void grouping_create_percentile50(RRDR *r, const char *options);
11 +extern void grouping_create_percentile75(RRDR *r, const char *options);
12 +extern void grouping_create_percentile80(RRDR *r, const char *options);
13 +extern void grouping_create_percentile90(RRDR *r, const char *options);
14 +extern void grouping_create_percentile95(RRDR *r, const char *options);
15 +extern void grouping_create_percentile97(RRDR *r, const char *options);
16 +extern void grouping_create_percentile98(RRDR *r, const char *options);
17 +extern void grouping_create_percentile99(RRDR *r, const char *options );
18 +extern void grouping_reset_percentile(RRDR *r);
19 +extern void grouping_free_percentile(RRDR *r);
20 +extern void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value);
21 +extern NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
22 +
23 +#endif //NETDATA_API_QUERIES_PERCENTILE_H
web/api/queries/query.c
+310
@@ -14,6 +14,8 @@
14 #include "stddev/stddev.h"
15 #include "ses/ses.h"
16 #include "des/des.h"
17 +#include "percentile/percentile.h"
18 +#include "trimmed_mean/trimmed_mean.h"
19
20 // ----------------------------------------------------------------------------
21
@@ -74,6 +76,105 @@ static struct {
76 .flush = grouping_flush_average,
77 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
78 },
79 + {.name = "trimmed-mean1",
80 + .hash = 0,
81 + .value = RRDR_GROUPING_TRIMMED_MEAN1,
82 + .init = NULL,
83 + .create= grouping_create_trimmed_mean1,
84 + .reset = grouping_reset_trimmed_mean,
85 + .free = grouping_free_trimmed_mean,
86 + .add = grouping_add_trimmed_mean,
87 + .flush = grouping_flush_trimmed_mean,
88 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
89 + },
90 + {.name = "trimmed-mean2",
91 + .hash = 0,
92 + .value = RRDR_GROUPING_TRIMMED_MEAN2,
93 + .init = NULL,
94 + .create= grouping_create_trimmed_mean2,
95 + .reset = grouping_reset_trimmed_mean,
96 + .free = grouping_free_trimmed_mean,
97 + .add = grouping_add_trimmed_mean,
98 + .flush = grouping_flush_trimmed_mean,
99 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
100 + },
101 + {.name = "trimmed-mean3",
102 + .hash = 0,
103 + .value = RRDR_GROUPING_TRIMMED_MEAN3,
104 + .init = NULL,
105 + .create= grouping_create_trimmed_mean3,
106 + .reset = grouping_reset_trimmed_mean,
107 + .free = grouping_free_trimmed_mean,
108 + .add = grouping_add_trimmed_mean,
109 + .flush = grouping_flush_trimmed_mean,
110 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
111 + },
112 + {.name = "trimmed-mean5",
113 + .hash = 0,
114 + .value = RRDR_GROUPING_TRIMMED_MEAN5,
115 + .init = NULL,
116 + .create= grouping_create_trimmed_mean5,
117 + .reset = grouping_reset_trimmed_mean,
118 + .free = grouping_free_trimmed_mean,
119 + .add = grouping_add_trimmed_mean,
120 + .flush = grouping_flush_trimmed_mean,
121 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
122 + },
123 + {.name = "trimmed-mean10",
124 + .hash = 0,
125 + .value = RRDR_GROUPING_TRIMMED_MEAN10,
126 + .init = NULL,
127 + .create= grouping_create_trimmed_mean10,
128 + .reset = grouping_reset_trimmed_mean,
129 + .free = grouping_free_trimmed_mean,
130 + .add = grouping_add_trimmed_mean,
131 + .flush = grouping_flush_trimmed_mean,
132 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
133 + },
134 + {.name = "trimmed-mean15",
135 + .hash = 0,
136 + .value = RRDR_GROUPING_TRIMMED_MEAN15,
137 + .init = NULL,
138 + .create= grouping_create_trimmed_mean15,
139 + .reset = grouping_reset_trimmed_mean,
140 + .free = grouping_free_trimmed_mean,
141 + .add = grouping_add_trimmed_mean,
142 + .flush = grouping_flush_trimmed_mean,
143 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
144 + },
145 + {.name = "trimmed-mean20",
146 + .hash = 0,
147 + .value = RRDR_GROUPING_TRIMMED_MEAN20,
148 + .init = NULL,
149 + .create= grouping_create_trimmed_mean20,
150 + .reset = grouping_reset_trimmed_mean,
151 + .free = grouping_free_trimmed_mean,
152 + .add = grouping_add_trimmed_mean,
153 + .flush = grouping_flush_trimmed_mean,
154 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
155 + },
156 + {.name = "trimmed-mean25",
157 + .hash = 0,
158 + .value = RRDR_GROUPING_TRIMMED_MEAN25,
159 + .init = NULL,
160 + .create= grouping_create_trimmed_mean25,
161 + .reset = grouping_reset_trimmed_mean,
162 + .free = grouping_free_trimmed_mean,
163 + .add = grouping_add_trimmed_mean,
164 + .flush = grouping_flush_trimmed_mean,
165 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
166 + },
167 + {.name = "trimmed-mean",
168 + .hash = 0,
169 + .value = RRDR_GROUPING_TRIMMED_MEAN5,
170 + .init = NULL,
171 + .create= grouping_create_trimmed_mean5,
172 + .reset = grouping_reset_trimmed_mean,
173 + .free = grouping_free_trimmed_mean,
174 + .add = grouping_add_trimmed_mean,
175 + .flush = grouping_flush_trimmed_mean,
176 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
177 + },
178 {.name = "incremental_sum",
179 .hash = 0,
180 .value = RRDR_GROUPING_INCREMENTAL_SUM,
@@ -107,6 +208,215 @@ static struct {
208 .flush = grouping_flush_median,
209 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
210 },
211 + {.name = "trimmed-median1",
212 + .hash = 0,
213 + .value = RRDR_GROUPING_TRIMMED_MEDIAN1,
214 + .init = NULL,
215 + .create= grouping_create_trimmed_median1,
216 + .reset = grouping_reset_median,
217 + .free = grouping_free_median,
218 + .add = grouping_add_median,
219 + .flush = grouping_flush_median,
220 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
221 + },
222 + {.name = "trimmed-median2",
223 + .hash = 0,
224 + .value = RRDR_GROUPING_TRIMMED_MEDIAN2,
225 + .init = NULL,
226 + .create= grouping_create_trimmed_median2,
227 + .reset = grouping_reset_median,
228 + .free = grouping_free_median,
229 + .add = grouping_add_median,
230 + .flush = grouping_flush_median,
231 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
232 + },
233 + {.name = "trimmed-median3",
234 + .hash = 0,
235 + .value = RRDR_GROUPING_TRIMMED_MEDIAN3,
236 + .init = NULL,
237 + .create= grouping_create_trimmed_median3,
238 + .reset = grouping_reset_median,
239 + .free = grouping_free_median,
240 + .add = grouping_add_median,
241 + .flush = grouping_flush_median,
242 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
243 + },
244 + {.name = "trimmed-median5",
245 + .hash = 0,
246 + .value = RRDR_GROUPING_TRIMMED_MEDIAN5,
247 + .init = NULL,
248 + .create= grouping_create_trimmed_median5,
249 + .reset = grouping_reset_median,
250 + .free = grouping_free_median,
251 + .add = grouping_add_median,
252 + .flush = grouping_flush_median,
253 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
254 + },
255 + {.name = "trimmed-median10",
256 + .hash = 0,
257 + .value = RRDR_GROUPING_TRIMMED_MEDIAN10,
258 + .init = NULL,
259 + .create= grouping_create_trimmed_median10,
260 + .reset = grouping_reset_median,
261 + .free = grouping_free_median,
262 + .add = grouping_add_median,
263 + .flush = grouping_flush_median,
264 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
265 + },
266 + {.name = "trimmed-median15",
267 + .hash = 0,
268 + .value = RRDR_GROUPING_TRIMMED_MEDIAN15,
269 + .init = NULL,
270 + .create= grouping_create_trimmed_median15,
271 + .reset = grouping_reset_median,
272 + .free = grouping_free_median,
273 + .add = grouping_add_median,
274 + .flush = grouping_flush_median,
275 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
276 + },
277 + {.name = "trimmed-median20",
278 + .hash = 0,
279 + .value = RRDR_GROUPING_TRIMMED_MEDIAN20,
280 + .init = NULL,
281 + .create= grouping_create_trimmed_median20,
282 + .reset = grouping_reset_median,
283 + .free = grouping_free_median,
284 + .add = grouping_add_median,
285 + .flush = grouping_flush_median,
286 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
287 + },
288 + {.name = "trimmed-median25",
289 + .hash = 0,
290 + .value = RRDR_GROUPING_TRIMMED_MEDIAN25,
291 + .init = NULL,
292 + .create= grouping_create_trimmed_median25,
293 + .reset = grouping_reset_median,
294 + .free = grouping_free_median,
295 + .add = grouping_add_median,
296 + .flush = grouping_flush_median,
297 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
298 + },
299 + {.name = "trimmed-median",
300 + .hash = 0,
301 + .value = RRDR_GROUPING_TRIMMED_MEDIAN5,
302 + .init = NULL,
303 + .create= grouping_create_trimmed_median5,
304 + .reset = grouping_reset_median,
305 + .free = grouping_free_median,
306 + .add = grouping_add_median,
307 + .flush = grouping_flush_median,
308 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
309 + },
310 + {.name = "percentile25",
311 + .hash = 0,
312 + .value = RRDR_GROUPING_PERCENTILE25,
313 + .init = NULL,
314 + .create= grouping_create_percentile25,
315 + .reset = grouping_reset_percentile,
316 + .free = grouping_free_percentile,
317 + .add = grouping_add_percentile,
318 + .flush = grouping_flush_percentile,
319 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
320 + },
321 + {.name = "percentile50",
322 + .hash = 0,
323 + .value = RRDR_GROUPING_PERCENTILE50,
324 + .init = NULL,
325 + .create= grouping_create_percentile50,
326 + .reset = grouping_reset_percentile,
327 + .free = grouping_free_percentile,
328 + .add = grouping_add_percentile,
329 + .flush = grouping_flush_percentile,
330 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
331 + },
332 + {.name = "percentile75",
333 + .hash = 0,
334 + .value = RRDR_GROUPING_PERCENTILE75,
335 + .init = NULL,
336 + .create= grouping_create_percentile75,
337 + .reset = grouping_reset_percentile,
338 + .free = grouping_free_percentile,
339 + .add = grouping_add_percentile,
340 + .flush = grouping_flush_percentile,
341 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
342 + },
343 + {.name = "percentile80",
344 + .hash = 0,
345 + .value = RRDR_GROUPING_PERCENTILE80,
346 + .init = NULL,
347 + .create= grouping_create_percentile80,
348 + .reset = grouping_reset_percentile,
349 + .free = grouping_free_percentile,
350 + .add = grouping_add_percentile,
351 + .flush = grouping_flush_percentile,
352 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
353 + },
354 + {.name = "percentile90",
355 + .hash = 0,
356 + .value = RRDR_GROUPING_PERCENTILE90,
357 + .init = NULL,
358 + .create= grouping_create_percentile90,
359 + .reset = grouping_reset_percentile,
360 + .free = grouping_free_percentile,
361 + .add = grouping_add_percentile,
362 + .flush = grouping_flush_percentile,
363 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
364 + },
365 + {.name = "percentile95",
366 + .hash = 0,
367 + .value = RRDR_GROUPING_PERCENTILE95,
368 + .init = NULL,
369 + .create= grouping_create_percentile95,
370 + .reset = grouping_reset_percentile,
371 + .free = grouping_free_percentile,
372 + .add = grouping_add_percentile,
373 + .flush = grouping_flush_percentile,
374 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
375 + },
376 + {.name = "percentile97",
377 + .hash = 0,
378 + .value = RRDR_GROUPING_PERCENTILE97,
379 + .init = NULL,
380 + .create= grouping_create_percentile97,
381 + .reset = grouping_reset_percentile,
382 + .free = grouping_free_percentile,
383 + .add = grouping_add_percentile,
384 + .flush = grouping_flush_percentile,
385 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
386 + },
387 + {.name = "percentile98",
388 + .hash = 0,
389 + .value = RRDR_GROUPING_PERCENTILE98,
390 + .init = NULL,
391 + .create= grouping_create_percentile98,
392 + .reset = grouping_reset_percentile,
393 + .free = grouping_free_percentile,
394 + .add = grouping_add_percentile,
395 + .flush = grouping_flush_percentile,
396 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
397 + },
398 + {.name = "percentile99",
399 + .hash = 0,
400 + .value = RRDR_GROUPING_PERCENTILE99,
401 + .init = NULL,
402 + .create= grouping_create_percentile99,
403 + .reset = grouping_reset_percentile,
404 + .free = grouping_free_percentile,
405 + .add = grouping_add_percentile,
406 + .flush = grouping_flush_percentile,
407 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
408 + },
409 + {.name = "percentile",
410 + .hash = 0,
411 + .value = RRDR_GROUPING_PERCENTILE95,
412 + .init = NULL,
413 + .create= grouping_create_percentile95,
414 + .reset = grouping_reset_percentile,
415 + .free = grouping_free_percentile,
416 + .add = grouping_add_percentile,
417 + .flush = grouping_flush_percentile,
418 + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
419 + },
420 {.name = "min",
421 .hash = 0,
422 .value = RRDR_GROUPING_MIN,
web/api/queries/query.h
+25
@@ -14,7 +14,32 @@ typedef enum rrdr_grouping {
14 RRDR_GROUPING_MAX,
15 RRDR_GROUPING_SUM,
16 RRDR_GROUPING_INCREMENTAL_SUM,
17 + RRDR_GROUPING_TRIMMED_MEAN1,
18 + RRDR_GROUPING_TRIMMED_MEAN2,
19 + RRDR_GROUPING_TRIMMED_MEAN3,
20 + RRDR_GROUPING_TRIMMED_MEAN5,
21 + RRDR_GROUPING_TRIMMED_MEAN10,
22 + RRDR_GROUPING_TRIMMED_MEAN15,
23 + RRDR_GROUPING_TRIMMED_MEAN20,
24 + RRDR_GROUPING_TRIMMED_MEAN25,
25 RRDR_GROUPING_MEDIAN,
26 + RRDR_GROUPING_TRIMMED_MEDIAN1,
27 + RRDR_GROUPING_TRIMMED_MEDIAN2,
28 + RRDR_GROUPING_TRIMMED_MEDIAN3,
29 + RRDR_GROUPING_TRIMMED_MEDIAN5,
30 + RRDR_GROUPING_TRIMMED_MEDIAN10,
31 + RRDR_GROUPING_TRIMMED_MEDIAN15,
32 + RRDR_GROUPING_TRIMMED_MEDIAN20,
33 + RRDR_GROUPING_TRIMMED_MEDIAN25,
34 + RRDR_GROUPING_PERCENTILE25,
35 + RRDR_GROUPING_PERCENTILE50,
36 + RRDR_GROUPING_PERCENTILE75,
37 + RRDR_GROUPING_PERCENTILE80,
38 + RRDR_GROUPING_PERCENTILE90,
39 + RRDR_GROUPING_PERCENTILE95,
40 + RRDR_GROUPING_PERCENTILE97,
41 + RRDR_GROUPING_PERCENTILE98,
42 + RRDR_GROUPING_PERCENTILE99,
43 RRDR_GROUPING_STDDEV,
44 RRDR_GROUPING_CV,
45 RRDR_GROUPING_SES,
web/api/queries/trimmed_mean/Makefile.am new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +dist_noinst_DATA = \
7 + README.md \
8 + $(NULL)
web/api/queries/trimmed_mean/README.md new
+56
@@ -0,0 +1,56 @@
1 +<!--
2 +title: "Trimmed Mean"
3 +description: "Use trimmed-mean in API queries and health entities to find the average value from a sample, eliminating any unwanted spikes in the returned metrics."
4 +custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/trimmed_mean/README.md
5 +-->
6 +
7 +# Trimmed Mean
8 +
9 +The trimmed mean is the average value of a series excluding the smallest and biggest points.
10 +
11 +Netdata applies linear interpolation on the last point, if the percentage requested to be excluded does not give a
12 +round number of points.
13 +
14 +The following percentile aliases are defined:
15 +
16 +- `trimmed-mean1`
17 +- `trimmed-mean2`
18 +- `trimmed-mean3`
19 +- `trimmed-mean5`
20 +- `trimmed-mean10`
21 +- `trimmed-mean15`
22 +- `trimmed-mean20`
23 +- `trimmed-mean25`
24 +
25 +The default `trimmed-mean` is an alias for `trimmed-mean5`.
26 +Any percentage may be requested using the `group_options` query parameter.
27 +
28 +## how to use
29 +
30 +Use it in alarms like this:
31 +
32 +```
33 + alarm: my_alarm
34 + on: my_chart
35 +lookup: trimmed-mean5 -1m unaligned of my_dimension
36 + warn: $this > 1000
37 +```
38 +
39 +`trimmed-mean` does not change the units. For example, if the chart units is `requests/sec`, the result
40 +will be again expressed in the same units.
41 +
42 +It can also be used in APIs and badges as `&group=trimmed-mean` in the URL and the additional parameter `group_options`
43 +may be used to request any percentage (e.g. `&group=trimmed-mean&group_options=29`).
44 +
45 +## Examples
46 +
47 +Examining last 1 minute `successful` web server responses:
48 +
49 +- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=min&after=-60&label=min)
50 +- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=average&after=-60&label=average)
51 +- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=trimmed-mean5&after=-60&label=trimmed-mean5&value_color=orange)
52 +- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=max&after=-60&label=max)
53 +
54 +## References
55 +
56 +- <https://en.wikipedia.org/wiki/Truncated_mean>.
web/api/queries/trimmed_mean/trimmed_mean.c new
+166
@@ -0,0 +1,166 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "trimmed_mean.h"
4 +
5 +// ----------------------------------------------------------------------------
6 +// median
7 +
8 +struct grouping_trimmed_mean {
9 + size_t series_size;
10 + size_t next_pos;
11 + NETDATA_DOUBLE percent;
12 +
13 + NETDATA_DOUBLE *series;
14 +};
15 +
16 +static void grouping_create_trimmed_mean_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
17 + long entries = r->group;
18 + if(entries < 10) entries = 10;
19 +
20 + struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_trimmed_mean));
21 + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
22 + g->series_size = (size_t)entries;
23 +
24 + g->percent = def;
25 + if(options && *options) {
26 + g->percent = str2ndd(options, NULL);
27 + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
28 + if(g->percent < 0.0) g->percent = 0.0;
29 + if(g->percent > 50.0) g->percent = 50.0;
30 + }
31 +
32 + g->percent = 1.0 - ((g->percent / 100.0) * 2.0);
33 + r->internal.grouping_data = g;
34 +}
35 +
36 +void grouping_create_trimmed_mean1(RRDR *r, const char *options) {
37 + grouping_create_trimmed_mean_internal(r, options, 1.0);
38 +}
39 +void grouping_create_trimmed_mean2(RRDR *r, const char *options) {
40 + grouping_create_trimmed_mean_internal(r, options, 2.0);
41 +}
42 +void grouping_create_trimmed_mean3(RRDR *r, const char *options) {
43 + grouping_create_trimmed_mean_internal(r, options, 3.0);
44 +}
45 +void grouping_create_trimmed_mean5(RRDR *r, const char *options) {
46 + grouping_create_trimmed_mean_internal(r, options, 5.0);
47 +}
48 +void grouping_create_trimmed_mean10(RRDR *r, const char *options) {
49 + grouping_create_trimmed_mean_internal(r, options, 10.0);
50 +}
51 +void grouping_create_trimmed_mean15(RRDR *r, const char *options) {
52 + grouping_create_trimmed_mean_internal(r, options, 15.0);
53 +}
54 +void grouping_create_trimmed_mean20(RRDR *r, const char *options) {
55 + grouping_create_trimmed_mean_internal(r, options, 20.0);
56 +}
57 +void grouping_create_trimmed_mean25(RRDR *r, const char *options) {
58 + grouping_create_trimmed_mean_internal(r, options, 25.0);
59 +}
60 +
61 +// resets when switches dimensions
62 +// so, clear everything to restart
63 +void grouping_reset_trimmed_mean(RRDR *r) {
64 + struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
65 + g->next_pos = 0;
66 +}
67 +
68 +void grouping_free_trimmed_mean(RRDR *r) {
69 + struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
70 + if(g) onewayalloc_freez(r->internal.owa, g->series);
71 +
72 + onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
73 + r->internal.grouping_data = NULL;
74 +}
75 +
76 +void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value) {
77 + struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
78 +
79 + if(unlikely(g->next_pos >= g->series_size)) {
80 + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
81 + g->series_size *= 2;
82 + }
83 +
84 + g->series[g->next_pos++] = value;
85 +}
86 +
87 +NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
88 + struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
89 +
90 + NETDATA_DOUBLE value;
91 + size_t available_slots = g->next_pos;
92 +
93 + if(unlikely(!available_slots)) {
94 + value = 0.0;
95 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
96 + }
97 + else if(available_slots == 1) {
98 + value = g->series[0];
99 + }
100 + else {
101 + sort_series(g->series, available_slots);
102 +
103 + NETDATA_DOUBLE min = g->series[0];
104 + NETDATA_DOUBLE max = g->series[available_slots - 1];
105 +
106 + if (min != max) {
107 + size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent);
108 + if(!slots_to_use) slots_to_use = 1;
109 +
110 + NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots;
111 + NETDATA_DOUBLE percent_delta = g->percent - percent_to_use;
112 +
113 + NETDATA_DOUBLE percent_interpolation_slot = 0.0;
114 + NETDATA_DOUBLE percent_last_slot = 0.0;
115 + if(percent_delta > 0.0) {
116 + NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots;
117 + NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use;
118 +
119 + percent_interpolation_slot = percent_delta / percent_1slot;
120 + percent_last_slot = 1 - percent_interpolation_slot;
121 + }
122 +
123 + int start_slot, stop_slot, step, last_slot, interpolation_slot;
124 + if(min >= 0.0 && max >= 0.0) {
125 + start_slot = (int)((available_slots - slots_to_use) / 2);
126 + stop_slot = start_slot + (int)slots_to_use;
127 + last_slot = stop_slot - 1;
128 + interpolation_slot = stop_slot;
129 + step = 1;
130 + }
131 + else {
132 + start_slot = (int)available_slots - 1 - (int)((available_slots - slots_to_use) / 2);
133 + stop_slot = start_slot - (int)slots_to_use;
134 + last_slot = stop_slot + 1;
135 + interpolation_slot = stop_slot;
136 + step = -1;
137 + }
138 +
139 + value = 0.0;
140 + for(int slot = start_slot; slot != stop_slot ; slot += step)
141 + value += g->series[slot];
142 +
143 + size_t counted = slots_to_use;
144 + if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) {
145 + value += g->series[interpolation_slot] * percent_interpolation_slot;
146 + value += g->series[last_slot] * percent_last_slot;
147 + counted++;
148 + }
149 +
150 + value = value / (NETDATA_DOUBLE)counted;
151 + }
152 + else
153 + value = min;
154 + }
155 +
156 + if(unlikely(!netdata_double_isnumber(value))) {
157 + value = 0.0;
158 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
159 + }
160 +
161 + //log_series_to_stderr(g->series, g->next_pos, value, "trimmed_mean");
162 +
163 + g->next_pos = 0;
164 +
165 + return value;
166 +}
web/api/queries/trimmed_mean/trimmed_mean.h new
+22
@@ -0,0 +1,22 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_API_QUERIES_TRIMMED_MEAN_H
4 +#define NETDATA_API_QUERIES_TRIMMED_MEAN_H
5 +
6 +#include "../query.h"
7 +#include "../rrdr.h"
8 +
9 +extern void grouping_create_trimmed_mean1(RRDR *r, const char *options);
10 +extern void grouping_create_trimmed_mean2(RRDR *r, const char *options);
11 +extern void grouping_create_trimmed_mean3(RRDR *r, const char *options);
12 +extern void grouping_create_trimmed_mean5(RRDR *r, const char *options);
13 +extern void grouping_create_trimmed_mean10(RRDR *r, const char *options);
14 +extern void grouping_create_trimmed_mean15(RRDR *r, const char *options);
15 +extern void grouping_create_trimmed_mean20(RRDR *r, const char *options);
16 +extern void grouping_create_trimmed_mean25(RRDR *r, const char *options);
17 +extern void grouping_reset_trimmed_mean(RRDR *r);
18 +extern void grouping_free_trimmed_mean(RRDR *r);
19 +extern void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value);
20 +extern NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
21 +
22 +#endif //NETDATA_API_QUERIES_TRIMMED_MEAN_H