@cryptotaxi247 / netdata-1 / commits / 49313a28b

refactor(go/plugin/framework/functions): redesign manager (#21850)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Ilya Mashchenko committed Mar 1, 2026 at 00:55 UTC 49313a28b0acd6d9a2b8fc76431e8b09f4d17b0d
32 files changed +3415 -268
src/go/plugin/agent/agent.go
+1
@@ -230,6 +230,7 @@ func (a *Agent) run(ctx context.Context) {
230 runtimeSvc := runtimechartemit.New(a.Logger.With(slog.String("component", "runtime metrics service")))
231 runtimeSvc.Start(a.Name, a.Out)
232 defer runtimeSvc.Stop()
233 + fnMgr.SetRuntimeService(runtimeSvc)
234
235 var runJob []string
236 if a.RunModule != "" && a.RunModule != "all" {
src/go/plugin/agent/discovery/sd/dyncfg.go
+2 -6
@@ -143,12 +143,8 @@ func (d *ServiceDiscovery) dyncfgConfig(fn dyncfg.Function) {
143 return
144 }
145
146 - // State-changing commands are queued for serial execution
147 - select {
148 - case <-d.ctx.Done():
149 - d.dyncfgApi.SendCodef(fn, 503, "Service discovery is shutting down.")
150 - case d.dyncfgCh <- fn:
151 - }
146 + // State-changing commands are queued for serial execution.
147 + d.enqueueDyncfgFunction(fn)
148 }
149
150 // dyncfgSeqExec executes state-changing dyncfg commands serially.
src/go/plugin/agent/discovery/sd/dyncfg_handoff.go new
+40
@@ -0,0 +1,40 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package sd
4 +
5 +import (
6 + "context"
7 +
8 + "github.com/netdata/netdata/go/plugins/plugin/framework/dyncfg"
9 +)
10 +
11 +const sdBusyMsg = "Service discovery is busy, try again later."
12 +
13 +func (d *ServiceDiscovery) enqueueDyncfgFunction(fn dyncfg.Function) {
14 + handoffCtx, cancel := d.dyncfgHandoffContext(fn)
15 + defer cancel()
16 +
17 + switch dyncfg.BoundedSend(handoffCtx, d.dyncfgCh, fn, dyncfg.DefaultDownstreamHandoffCap) {
18 + case dyncfg.BoundedSendOK:
19 + return
20 + case dyncfg.BoundedSendContextDone:
21 + if d.ctx != nil && d.ctx.Err() != nil {
22 + d.dyncfgApi.SendCodef(fn, 503, "Service discovery is shutting down.")
23 + return
24 + }
25 + d.dyncfgApi.SendCodef(fn, 503, sdBusyMsg)
26 + case dyncfg.BoundedSendTimeout:
27 + d.dyncfgApi.SendCodef(fn, 503, sdBusyMsg)
28 + }
29 +}
30 +
31 +func (d *ServiceDiscovery) dyncfgHandoffContext(fn dyncfg.Function) (context.Context, context.CancelFunc) {
32 + ctx := d.ctx
33 + if ctx == nil {
34 + ctx = context.Background()
35 + }
36 + if timeout := fn.Fn().Timeout; timeout > 0 {
37 + return context.WithTimeout(ctx, timeout)
38 + }
39 + return ctx, func() {}
40 +}
src/go/plugin/agent/discovery/sd/sd.go
+8
@@ -58,6 +58,11 @@ func NewServiceDiscovery(cfg Config) (*ServiceDiscovery, error) {
58 exposed: dyncfg.NewExposedCache[sdConfig](),
59 dyncfgCh: make(chan dyncfg.Function, 1),
60 }
61 + if provider, ok := cfg.FnReg.(interface {
62 + TerminalFinalizer() functions.TerminalFinalizer
63 + }); ok {
64 + d.dyncfgApi.SetTerminalFinalizer(provider.TerminalFinalizer())
65 + }
66 d.newPipeline = func(config pipeline.Config) (sdPipeline, error) {
67 return pipeline.New(config, d.newDiscoverersFromRegistry)
68 }
@@ -122,6 +127,9 @@ type (
127
128 // SetDyncfgResponder allows overriding the default responder (e.g., to silence output in tests).
129 func (d *ServiceDiscovery) SetDyncfgResponder(api *dyncfg.Responder) {
130 + if api != nil && d.dyncfgApi != nil {
131 + api.SetTerminalFinalizer(d.dyncfgApi.TerminalFinalizer())
132 + }
133 dyncfg.BindResponder(&d.dyncfgApi, d.handler, api)
134 }
135
src/go/plugin/agent/jobmgr/dyncfg_collector.go
+1 -6
@@ -85,12 +85,7 @@ func (m *Manager) dyncfgCollectorExec(fn dyncfg.Function) {
85 m.dyncfgCmdSchema(fn)
86 return
87 }
88 -
89 - select {
90 - case <-m.ctx.Done():
91 - m.dyncfgApi.SendCodef(fn, 503, "Job manager is shutting down.")
92 - case m.dyncfgCh <- fn:
93 - }
88 + m.enqueueDyncfgFunction(fn)
89 }
90
91 func (m *Manager) dyncfgCollectorSeqExec(fn dyncfg.Function) {
src/go/plugin/agent/jobmgr/dyncfg_handoff.go new
+40
@@ -0,0 +1,40 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package jobmgr
4 +
5 +import (
6 + "context"
7 +
8 + "github.com/netdata/netdata/go/plugins/plugin/framework/dyncfg"
9 +)
10 +
11 +const (
12 + dyncfgBusyMsg = "Job manager is busy, try again later."
13 + dyncfgShuttingDownMsg = "Job manager is shutting down."
14 +)
15 +
16 +func (m *Manager) enqueueDyncfgFunction(fn dyncfg.Function) {
17 + handoffCtx, cancel := m.dyncfgHandoffContext(fn)
18 + defer cancel()
19 +
20 + switch dyncfg.BoundedSend(handoffCtx, m.dyncfgCh, fn, dyncfg.DefaultDownstreamHandoffCap) {
21 + case dyncfg.BoundedSendOK:
22 + return
23 + case dyncfg.BoundedSendContextDone:
24 + if m.baseContext().Err() != nil {
25 + m.dyncfgApi.SendCodef(fn, 503, dyncfgShuttingDownMsg)
26 + return
27 + }
28 + m.dyncfgApi.SendCodef(fn, 503, dyncfgBusyMsg)
29 + case dyncfg.BoundedSendTimeout:
30 + m.dyncfgApi.SendCodef(fn, 503, dyncfgBusyMsg)
31 + }
32 +}
33 +
34 +func (m *Manager) dyncfgHandoffContext(fn dyncfg.Function) (context.Context, context.CancelFunc) {
35 + ctx := m.baseContext()
36 + if timeout := fn.Fn().Timeout; timeout > 0 {
37 + return context.WithTimeout(ctx, timeout)
38 + }
39 + return ctx, func() {}
40 +}
src/go/plugin/agent/jobmgr/dyncfg_vnode.go
+1 -6
@@ -82,12 +82,7 @@ func (m *Manager) dyncfgVnodeExec(fn dyncfg.Function) {
82 m.dyncfgApi.SendJSON(fn, vnodes.ConfigSchema)
83 return
84 }
85 -
86 - select {
87 - case <-m.ctx.Done():
88 - m.dyncfgApi.SendCodef(fn, 503, "Job manager is shutting down.")
89 - case m.dyncfgCh <- fn:
90 - }
85 + m.enqueueDyncfgFunction(fn)
86 }
87
88 func (m *Manager) dyncfgVnodeSeqExec(fn dyncfg.Function) {
src/go/plugin/agent/jobmgr/manager.go
+8
@@ -63,6 +63,11 @@ func New(cfg Config) *Manager {
63 if fnReg == nil {
64 fnReg = noop{}
65 }
66 + if provider, ok := fnReg.(interface {
67 + TerminalFinalizer() functions.TerminalFinalizer
68 + }); ok {
69 + api.SetTerminalFinalizer(provider.TerminalFinalizer())
70 + }
71 vnodesReg := cfg.Vnodes
72 if vnodesReg == nil {
73 vnodesReg = make(map[string]*vnodes.VirtualNode)
@@ -135,6 +140,9 @@ func New(cfg Config) *Manager {
140
141 // SetDyncfgResponder allows overriding the default responder (e.g., to silence output in CLI mode).
142 func (m *Manager) SetDyncfgResponder(responder *dyncfg.Responder) {
143 + if responder != nil && m.dyncfgApi != nil {
144 + responder.SetTerminalFinalizer(m.dyncfgApi.TerminalFinalizer())
145 + }
146 dyncfg.BindResponder(&m.dyncfgApi, m.handler, responder)
147 }
148
src/go/plugin/agent/runtimechartemit/README.md new
+37
@@ -0,0 +1,37 @@
1 +# runtimechartemit
2 +
3 +This package is the runtime/internal metrics bridge between component-owned
4 +`metrix.RuntimeStore` writers and Netdata chart protocol output.
5 +
6 +## Registration flow
7 +
8 +1. Component code creates/owns a `metrix.RuntimeStore` (writer side).
9 +2. Component registers itself via `runtimecomp.Service.RegisterComponent(...)` with:
10 + - stable `Name`
11 + - `Store`
12 + - optional `TemplateYAML` (or `Autogen.Enabled=true` for fallback template)
13 + - cadence metadata (`UpdateEvery`) and emit env metadata (`TypeID`, `Plugin`, `Module`, `JobName`, `JobLabels`).
14 +3. Service normalizes config and upserts it into the internal component registry.
15 +4. Runtime metrics job snapshots registry entries on each tick.
16 +5. For each component due on this tick:
17 + - read component store via `Read(metrix.ReadRaw(), metrix.ReadFlatten())`
18 + - build plan with chartengine
19 + - emit plan through `chartemit.ApplyPlan(...)`.
20 +6. When component is unregistered, runtime job emits obsolete/remove actions for previously known charts.
21 +
22 +## Lifecycle ownership
23 +
24 +- `Service.Start(pluginName, out)` starts runtime metrics job and cadence ticker.
25 +- `Service.Stop()` stops ticker and runtime job.
26 +- Components should register on start and unregister on stop to avoid stale runtime charts.
27 +
28 +## Producers vs components
29 +
30 +- Components: register a runtime store to be charted.
31 +- Producers: register a `tickFn` callback via `RegisterProducer` when a runtime source has no independent owner loop and must be advanced by runtime service cadence.
32 +
33 +## Operational notes
34 +
35 +- Runtime metrics job is intentionally single-flight; overlapping ticks are skipped and logged.
36 +- Observer chartengine runs with `WithRuntimeStore(nil)` (no self-instrumentation loop).
37 +- `Name` is the registry identity; re-registering the same name replaces generation and reinitializes runtime chart state.
src/go/plugin/framework/chartengine/runtime_metrics.go
+1 -1
@@ -102,7 +102,7 @@ func newRuntimeMetrics(store metrix.RuntimeStore) *runtimeMetrics {
102 if store == nil {
103 return nil
104 }
105 - meter := store.Write().StatefulMeter("netdata.go.plugin.chartengine")
105 + meter := store.Write().StatefulMeter("netdata.go.plugin.framework.chartengine")
106 phaseDuration := meter.Vec("phase").Summary(
107 "build_phase_duration_seconds",
108 metrix.WithSummaryQuantiles(0.5, 0.9, 0.99),
src/go/plugin/framework/chartengine/runtime_metrics_test.go
+20 -20
@@ -42,20 +42,20 @@ func TestEngineRuntimeObservabilityScenarios(t *testing.T) {
42 require.NotNil(t, rs)
43 r := rs.Read(metrix.ReadRaw())
44
45 - assertMetricValueAtLeast(t, r, "netdata.go.plugin.chartengine.build_success_total", nil, 2)
46 - assertSummaryCountAtLeast(t, r, "netdata.go.plugin.chartengine.build_duration_seconds", nil, 2)
47 - assertSummaryCountAtLeast(t, r, "netdata.go.plugin.chartengine.build_phase_duration_seconds", metrix.Labels{"phase": "scan"}, 2)
48 - assertMetricValueAtLeast(t, r, "netdata.go.plugin.chartengine.route_cache_misses_total", nil, 1)
49 - assertMetricValueAtLeast(t, r, "netdata.go.plugin.chartengine.route_cache_hits_total", nil, 1)
50 - assertMetricValueAtLeast(t, r, "netdata.go.plugin.chartengine.route_cache_entries", nil, 1)
51 - assertMetricValueAtLeast(t, r, "netdata.go.plugin.chartengine.route_cache_retained_total", nil, 1)
52 - assertMetricValueAtLeast(t, r, "netdata.go.plugin.chartengine.series_scanned_total", nil, 2)
53 - assertMetricValueAtLeast(t, r, "netdata.go.plugin.chartengine.planner_actions_total", metrix.Labels{"kind": "update_chart"}, 2)
54 - assertMetricValueAtLeast(t, r, "netdata.go.plugin.chartengine.plan_chart_instances", nil, 1)
45 + assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.build_success_total", nil, 2)
46 + assertSummaryCountAtLeast(t, r, "netdata.go.plugin.framework.chartengine.build_duration_seconds", nil, 2)
47 + assertSummaryCountAtLeast(t, r, "netdata.go.plugin.framework.chartengine.build_phase_duration_seconds", metrix.Labels{"phase": "scan"}, 2)
48 + assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.route_cache_misses_total", nil, 1)
49 + assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.route_cache_hits_total", nil, 1)
50 + assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.route_cache_entries", nil, 1)
51 + assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.route_cache_retained_total", nil, 1)
52 + assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.series_scanned_total", nil, 2)
53 + assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.planner_actions_total", metrix.Labels{"kind": "update_chart"}, 2)
54 + assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.plan_chart_instances", nil, 1)
55 assertMetricMeta(
56 t,
57 r,
58 - "netdata.go.plugin.chartengine.build_success_total",
58 + "netdata.go.plugin.framework.chartengine.build_success_total",
59 metrix.MetricMeta{
60 Description: "Successful BuildPlan calls",
61 ChartFamily: "ChartEngine/Build",
@@ -65,7 +65,7 @@ func TestEngineRuntimeObservabilityScenarios(t *testing.T) {
65 assertMetricMeta(
66 t,
67 r,
68 - "netdata.go.plugin.chartengine.planner_actions_total",
68 + "netdata.go.plugin.framework.chartengine.planner_actions_total",
69 metrix.MetricMeta{
70 Description: "Planner actions by kind",
71 ChartFamily: "ChartEngine/Actions",
@@ -91,7 +91,7 @@ func TestEngineRuntimeObservabilityScenarios(t *testing.T) {
91 require.NoError(t, err)
92
93 before := e.RuntimeStore().Read(metrix.ReadRaw())
94 - beforeCharts, ok := before.Value("netdata.go.plugin.chartengine.plan_chart_instances", nil)
94 + beforeCharts, ok := before.Value("netdata.go.plugin.framework.chartengine.plan_chart_instances", nil)
95 require.True(t, ok)
96 require.GreaterOrEqual(t, beforeCharts, float64(1))
97
@@ -101,10 +101,10 @@ func TestEngineRuntimeObservabilityScenarios(t *testing.T) {
101 require.NoError(t, err)
102
103 after := e.RuntimeStore().Read(metrix.ReadRaw())
104 - assertMetricValueAtLeast(t, after, "netdata.go.plugin.chartengine.build_skipped_failed_collect_total", nil, 1)
105 - assertMetricValueAtLeast(t, after, "netdata.go.plugin.chartengine.build_success_total", nil, 1)
104 + assertMetricValueAtLeast(t, after, "netdata.go.plugin.framework.chartengine.build_skipped_failed_collect_total", nil, 1)
105 + assertMetricValueAtLeast(t, after, "netdata.go.plugin.framework.chartengine.build_success_total", nil, 1)
106
107 - afterCharts, ok := after.Value("netdata.go.plugin.chartengine.plan_chart_instances", nil)
107 + afterCharts, ok := after.Value("netdata.go.plugin.framework.chartengine.plan_chart_instances", nil)
108 require.True(t, ok)
109 assert.Equal(t, beforeCharts, afterCharts)
110 },
@@ -129,7 +129,7 @@ func TestEngineRuntimeObservabilityScenarios(t *testing.T) {
129 require.NoError(t, err)
130
131 r := e.RuntimeStore().Read(metrix.ReadRaw())
132 - assertMetricValueAtLeast(t, r, "netdata.go.plugin.chartengine.series_filtered_total", metrix.Labels{"reason": "by_selector"}, 1)
132 + assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.series_filtered_total", metrix.Labels{"reason": "by_selector"}, 1)
133 },
134 },
135 "expiry removals are counted by scope and reason": {
@@ -162,7 +162,7 @@ func TestEngineRuntimeObservabilityScenarios(t *testing.T) {
162 assertMetricValueAtLeast(
163 t,
164 r,
165 - "netdata.go.plugin.chartengine.lifecycle_removed_total",
165 + "netdata.go.plugin.framework.chartengine.lifecycle_removed_total",
166 metrix.Labels{"scope": "dimension", "reason": "expiry"},
167 1,
168 )
@@ -226,13 +226,13 @@ func TestEngineRuntimeObservabilityScenarios(t *testing.T) {
226 assert.Equal(t, "ChartEngine/Build", create.Meta.Family)
227 assert.Equal(t, "builds/s", create.Meta.Units)
228
229 - duration := findCreateChartByID(plan.Actions, "netdata.go.plugin.chartengine.build_duration_seconds")
229 + duration := findCreateChartByID(plan.Actions, "netdata.go.plugin.framework.chartengine.build_duration_seconds")
230 require.NotNil(t, duration)
231 assert.Equal(t, "BuildPlan duration in seconds", duration.Meta.Title)
232 assert.Equal(t, "ChartEngine/Build", duration.Meta.Family)
233 assert.Equal(t, "seconds", duration.Meta.Units)
234
235 - durationSum := findCreateChartByID(plan.Actions, "netdata.go.plugin.chartengine.build_duration_seconds_sum")
235 + durationSum := findCreateChartByID(plan.Actions, "netdata.go.plugin.framework.chartengine.build_duration_seconds_sum")
236 require.NotNil(t, durationSum)
237 assert.Equal(t, "BuildPlan duration in seconds", durationSum.Meta.Title)
238 assert.Equal(t, "ChartEngine/Build", durationSum.Meta.Family)
src/go/plugin/framework/dyncfg/handoff.go new
+55
@@ -0,0 +1,55 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package dyncfg
4 +
5 +import (
6 + "context"
7 + "time"
8 +)
9 +
10 +const DefaultDownstreamHandoffCap = time.Second
11 +
12 +type BoundedSendResult uint8
13 +
14 +const (
15 + BoundedSendOK BoundedSendResult = iota + 1
16 + BoundedSendContextDone
17 + BoundedSendTimeout
18 +)
19 +
20 +// BoundedSend sends value to ch using bounded wait:
21 +// wait = min(remaining request deadline, maxWait), with maxWait used when no deadline exists.
22 +func BoundedSend[T any](ctx context.Context, ch chan<- T, value T, maxWait time.Duration) BoundedSendResult {
23 + if maxWait <= 0 {
24 + maxWait = DefaultDownstreamHandoffCap
25 + }
26 + if ctx == nil {
27 + ctx = context.Background()
28 + }
29 +
30 + wait := maxWait
31 + if deadline, ok := ctx.Deadline(); ok {
32 + remaining := time.Until(deadline)
33 + if remaining <= 0 {
34 + if ctx.Err() != nil {
35 + return BoundedSendContextDone
36 + }
37 + return BoundedSendTimeout
38 + }
39 + if remaining < wait {
40 + wait = remaining
41 + }
42 + }
43 +
44 + timer := time.NewTimer(wait)
45 + defer timer.Stop()
46 +
47 + select {
48 + case <-ctx.Done():
49 + return BoundedSendContextDone
50 + case ch <- value:
51 + return BoundedSendOK
52 + case <-timer.C:
53 + return BoundedSendTimeout
54 + }
55 +}
src/go/plugin/framework/dyncfg/handoff_test.go new
+83
@@ -0,0 +1,83 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package dyncfg
4 +
5 +import (
6 + "context"
7 + "testing"
8 + "time"
9 +
10 + "github.com/stretchr/testify/assert"
11 +)
12 +
13 +func TestBoundedSend(t *testing.T) {
14 + tests := map[string]struct {
15 + run func(t *testing.T) BoundedSendResult
16 + want BoundedSendResult
17 + }{
18 + "buffered channel send succeeds": {
19 + run: func(t *testing.T) BoundedSendResult {
20 + t.Helper()
21 + ch := make(chan int, 1)
22 + got := BoundedSend(context.Background(), ch, 42, 50*time.Millisecond)
23 + assert.Equal(t, 42, <-ch)
24 + return got
25 + },
26 + want: BoundedSendOK,
27 + },
28 + "unbuffered channel send succeeds with receiver": {
29 + run: func(t *testing.T) BoundedSendResult {
30 + t.Helper()
31 + ch := make(chan int)
32 + received := make(chan int, 1)
33 + go func() { received <- <-ch }()
34 + got := BoundedSend(context.Background(), ch, 77, 50*time.Millisecond)
35 + assert.Equal(t, 77, <-received)
36 + return got
37 + },
38 + want: BoundedSendOK,
39 + },
40 + "context canceled before send returns context-done": {
41 + run: func(t *testing.T) BoundedSendResult {
42 + t.Helper()
43 + ctx, cancel := context.WithCancel(context.Background())
44 + cancel()
45 + ch := make(chan int)
46 + return BoundedSend(ctx, ch, 1, 50*time.Millisecond)
47 + },
48 + want: BoundedSendContextDone,
49 + },
50 + "nil context uses background and times out": {
51 + run: func(t *testing.T) BoundedSendResult {
52 + t.Helper()
53 + ch := make(chan int)
54 + return BoundedSend[int](nil, ch, 1, 20*time.Millisecond)
55 + },
56 + want: BoundedSendTimeout,
57 + },
58 + "unbuffered channel without receiver times out": {
59 + run: func(t *testing.T) BoundedSendResult {
60 + t.Helper()
61 + ch := make(chan int)
62 + return BoundedSend(context.Background(), ch, 1, 20*time.Millisecond)
63 + },
64 + want: BoundedSendTimeout,
65 + },
66 + "expired context deadline returns context-done": {
67 + run: func(t *testing.T) BoundedSendResult {
68 + t.Helper()
69 + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
70 + defer cancel()
71 + ch := make(chan int)
72 + return BoundedSend(ctx, ch, 1, 50*time.Millisecond)
73 + },
74 + want: BoundedSendContextDone,
75 + },
76 + }
77 +
78 + for name, tc := range tests {
79 + t.Run(name, func(t *testing.T) {
80 + assert.Equal(t, tc.want, tc.run(t))
81 + })
82 + }
83 +}
src/go/plugin/framework/dyncfg/responder.go
+48 -23
@@ -3,22 +3,55 @@
3 package dyncfg
4
5 import (
6 - "encoding/json"
6 "fmt"
7 "strconv"
8 + "sync"
9 "time"
10
11 "github.com/netdata/netdata/go/plugins/pkg/netdataapi"
12 + fnpkg "github.com/netdata/netdata/go/plugins/plugin/framework/functions"
13 )
14
15 // Responder handles standardized responses for dyncfg operations
16 type Responder struct {
17 api *netdataapi.API
18 +
19 + finalizeMux sync.RWMutex
20 + finalize fnpkg.TerminalFinalizer
21 }
22
23 // NewResponder creates a new responder
24 func NewResponder(api *netdataapi.API) *Responder {
21 - return &Responder{api: api}
25 + return &Responder{
26 + api: api,
27 + finalize: fnpkg.DirectTerminalFinalizer,
28 + }
29 +}
30 +
31 +// SetTerminalFinalizer overrides terminal response finalization behavior.
32 +func (r *Responder) SetTerminalFinalizer(finalize fnpkg.TerminalFinalizer) {
33 + r.finalizeMux.Lock()
34 + defer r.finalizeMux.Unlock()
35 +
36 + if finalize == nil {
37 + r.finalize = fnpkg.DirectTerminalFinalizer
38 + return
39 + }
40 + r.finalize = finalize
41 +}
42 +
43 +// TerminalFinalizer returns the currently configured terminal finalizer.
44 +func (r *Responder) TerminalFinalizer() fnpkg.TerminalFinalizer {
45 + r.finalizeMux.RLock()
46 + defer r.finalizeMux.RUnlock()
47 + return r.finalize
48 +}
49 +
50 +func (r *Responder) finalizeTerminal(uid, source string, emit func()) bool {
51 + r.finalizeMux.RLock()
52 + finalize := r.finalize
53 + r.finalizeMux.RUnlock()
54 + return finalize(uid, source, emit)
55 }
56
57 // SendCodef sends a response with a specific code and message
@@ -32,31 +65,17 @@ func (r *Responder) SendCodef(fn Function, code int, message string, args ...any
65 msg = fmt.Sprintf(message, args...)
66 }
67
35 - var payload []byte
36 - if code >= 400 && code < 600 {
37 - payload, _ = json.Marshal(struct {
38 - Status int `json:"status"`
39 - ErrorMessage string `json:"errorMessage"`
40 - }{
41 - Status: code,
42 - ErrorMessage: msg,
43 - })
44 - } else {
45 - payload, _ = json.Marshal(struct {
46 - Status int `json:"status"`
47 - Message string `json:"message"`
48 - }{
49 - Status: code,
50 - Message: msg,
51 - })
52 - }
68 + payload := fnpkg.BuildJSONPayload(code, msg)
69
54 - r.api.FUNCRESULT(netdataapi.FunctionResult{
70 + res := netdataapi.FunctionResult{
71 UID: fn.UID(),
72 ContentType: "application/json",
73 Payload: string(payload),
74 Code: strconv.Itoa(code),
75 ExpireTimestamp: strconv.FormatInt(time.Now().Unix(), 10),
76 + }
77 + r.finalizeTerminal(fn.UID(), "dyncfg.responder.sendcodef", func() {
78 + r.api.FUNCRESULT(res)
79 })
80 }
81
@@ -71,12 +90,15 @@ func (r *Responder) SendJSONWithCode(fn Function, payload string, code int) {
90 return
91 }
92
74 - r.api.FUNCRESULT(netdataapi.FunctionResult{
93 + res := netdataapi.FunctionResult{
94 UID: fn.UID(),
95 ContentType: "application/json",
96 Payload: payload,
97 Code: strconv.Itoa(code),
98 ExpireTimestamp: strconv.FormatInt(time.Now().Unix(), 10),
99 + }
100 + r.finalizeTerminal(fn.UID(), "dyncfg.responder.sendjsonwithcode", func() {
101 + r.api.FUNCRESULT(res)
102 })
103 }
104
@@ -91,12 +113,15 @@ func (r *Responder) sendPayload(fn Function, payload, contentType string) {
113 return
114 }
115
94 - r.api.FUNCRESULT(netdataapi.FunctionResult{
116 + res := netdataapi.FunctionResult{
117 UID: fn.UID(),
118 ContentType: contentType,
119 Payload: payload,
120 Code: "200",
121 ExpireTimestamp: strconv.FormatInt(time.Now().Unix(), 10),
122 + }
123 + r.finalizeTerminal(fn.UID(), "dyncfg.responder.sendpayload", func() {
124 + r.api.FUNCRESULT(res)
125 })
126 }
127
src/go/plugin/framework/functions/README.md new
+219
@@ -0,0 +1,219 @@
1 +# framework/functions manager
2 +
3 +This document describes how the functions manager works **today** (current implementation).
4 +
5 +## Scope
6 +
7 +- Package: `src/go/plugin/framework/functions`
8 +- Main implementation:
9 + - `manager.go`
10 + - `manager_worker.go`
11 + - `scheduler.go`
12 + - `parser.go`
13 + - `finalizer.go`
14 +
15 +## Input protocol handled by parser
16 +
17 +The parser recognizes these line types:
18 +
19 +- `FUNCTION ...`
20 +- `FUNCTION_PAYLOAD ...` + payload body + `FUNCTION_PAYLOAD_END`
21 +- `FUNCTION_CANCEL <transaction_id>`
22 +- `FUNCTION_PROGRESS ...` (recognized/no-op event for manager)
23 +- `QUIT`
24 +
25 +Payload-mode control behavior:
26 +
27 +- `FUNCTION_CANCEL <same payload uid>`:
28 + - abort payload frame
29 + - emit pre-admission cancel event
30 +- `FUNCTION_CANCEL <different uid>`:
31 + - emit cancel event
32 + - continue payload accumulation
33 +- `FUNCTION_PROGRESS ...`:
34 + - emit progress event
35 + - continue payload accumulation
36 +- `QUIT`:
37 + - abort payload frame
38 + - emit quit event
39 +- Any other `FUNCTION*` control line:
40 + - abort current payload frame
41 + - never dispatch partial payload
42 +
43 +## Runtime architecture
44 +
45 +### Dispatcher
46 +
47 +The dispatcher loop:
48 +
49 +- reads input lines
50 +- parses events
51 +- handles cancel/quit/progress
52 +- resolves handler + route-aware schedule key
53 +- admits calls into keyed scheduler
54 +
55 +Admission checks:
56 +
57 +- manager stopping -> reject `503`
58 +- unknown/nil handler -> reject `501`
59 +- duplicate active/tombstoned UID -> ignore duplicate input (debug/warn log, no terminal output)
60 +- queue full -> reject `503`
61 +
62 +### Keyed scheduler + worker pool
63 +
64 +- fixed-size worker pool (`defaultWorkerCount = 1`)
65 +- bounded pending budget (`defaultQueueSize = 64`)
66 +- per-key serialization:
67 + - same schedule key executes sequentially
68 + - different schedule keys execute concurrently (up to worker count)
69 +- schedule key is route-aware:
70 + - direct registration: `fn.Name`
71 + - prefix registration: `fn.Name|<matched-prefix>`
72 +- prefix registration guard:
73 + - overlapped prefixes for the same function name are rejected at registration time
74 + - manager logs an error and keeps the previously registered prefix set unchanged
75 +- worker transitions lifecycle:
76 + - `queued -> running -> awaiting_result`
77 +- worker return is **not** terminal completion
78 +- panic path finalizes terminal `500`
79 +
80 +### Tracking and finalization
81 +
82 +Active requests are tracked by UID:
83 +
84 +- `invState` map (active entries)
85 +- tombstones (`defaultTombstoneTTL = 60s`) to block immediate UID reuse
86 +
87 +All terminal outputs go through:
88 +
89 +- manager-bound terminal finalizer (`m.finalizeTerminal`)
90 +- dyncfg responders receive manager finalizer wiring at component construction time
91 +
92 +`tryFinalize` guarantees:
93 +
94 +- first terminal writer wins
95 +- late terminal duplicates are dropped
96 +- fallback timer is stopped on finalization
97 +- awaiting-result warning timer is stopped on finalization
98 +- UID becomes tombstoned for a short window
99 +
100 +Awaiting-result observability:
101 +
102 +- when a worker returns without terminal output, manager moves UID to `awaiting_result`
103 +- manager starts a warning timer (`defaultAwaitingWarnDelay = 30s`, capped by function timeout if lower)
104 +- timer emits a warning log if UID is still `awaiting_result` (diagnostic only, no forced finalize)
105 +
106 +## Runtime metrics
107 +
108 +Functions manager owns an internal runtime store (`metrix.NewRuntimeStore()`), and
109 +can register it as a runtime component when runtime service is injected via
110 +`SetRuntimeService(...)`.
111 +
112 +Registered component metadata:
113 +
114 +- component name: `functions.manager`
115 +- module: `functions`
116 +- job: `manager`
117 +- autogen charts: enabled
118 +
119 +Pathology-focused metrics currently exposed:
120 +
121 +- gauges:
122 + - `netdata.go.plugin.framework.functions.manager.invocations_active`
123 + - `netdata.go.plugin.framework.functions.manager.invocations_awaiting_result`
124 + - `netdata.go.plugin.framework.functions.manager.scheduler_pending`
125 +- counters:
126 + - `netdata.go.plugin.framework.functions.manager.queue_full_total`
127 + - `netdata.go.plugin.framework.functions.manager.cancel_fallback_total`
128 + - `netdata.go.plugin.framework.functions.manager.late_terminal_dropped_total`
129 + - `netdata.go.plugin.framework.functions.manager.duplicate_uid_ignored_total`
130 +
131 +## Cancellation semantics
132 +
133 +### 1) Queued request
134 +
135 +- mark cancel requested
136 +- cancel internal context
137 +- finalize exactly once with `499`
138 +- worker skips execution if it dequeues a canceled request
139 +
140 +### 2) Running / awaiting_result request
141 +
142 +- mark cancel requested
143 +- call internal cancel func
144 +- start fallback timer (`defaultCancelFallbackDelay = 5s`)
145 +- if no terminal output arrives before timer, manager finalizes with `499`
146 +
147 +Important limitation:
148 +
149 +- handlers are currently `func(Function)` (no `context.Context` parameter)
150 +- manager cannot force-stop handler code directly
151 +- fallback `499` is the deterministic safety net
152 +
153 +### 3) Unknown / already completed request
154 +
155 +- no-op
156 +- debug log only
157 +
158 +## Shutdown behavior
159 +
160 +Shutdown uses one bounded path for `ctx.Done()`, `QUIT`, and input close (EOF):
161 +
162 +- set stopping
163 +- stop scheduler admission
164 +- wait up to `defaultShutdownDrainTimeout = 8s` for natural drain
165 +- if drain times out **or** unresolved active UIDs remain after worker drain:
166 + - cancel in-flight
167 + - force-finalize unresolved UIDs with `499`
168 + - hard-stop scheduler waiters
169 +
170 +Input close still enters the same bounded path above:
171 +
172 +- input close/EOF:
173 + - stop scheduler admission
174 + - attempt bounded drain first
175 + - escalate to cancel/force-finalize on timeout or if active UIDs remain unresolved
176 +
177 +## Flow diagram
178 +
179 +```mermaid
180 +flowchart TD
181 + A["Input line"] --> B["Parser.parseEvent()"]
182 + B -->|call| C["dispatchInvocation()"]
183 + B -->|cancel| D["handleCancelEvent()"]
184 + B -->|progress| E["No-op"]
185 + B -->|quit| F["Shutdown(canceling)"]
186 + B -->|parse error| G["Warn + continue"]
187 + C --> C1{"Admission checks"}
188 + C1 -->|stopping| R503["respf 503"]
189 + C1 -->|unregistered/nil handler| R501["respf 501"]
190 + C1 -->|duplicate/tombstoned UID| DUP["ignore duplicate + log"]
191 + C1 -->|queue full| R503
192 + C1 -->|accepted| Q["scheduler.enqueue by route key + state=queued"]
193 + Q --> S["keyScheduler"]
194 + S -->|same key busy| SQ["lane queue (serialized)"]
195 + S -->|key free| W["Worker"]
196 + W --> W1{"Start allowed?"}
197 + W1 -->|ctx canceled / cancelRequested| X["skip"]
198 + W1 -->|yes| W2["state=running; run handler"]
199 + W2 -->|panic| R500["respf 500"]
200 + W2 -->|return| AWAIT["state=awaiting_result"]
201 + D --> D1{"Cancel target state"}
202 + D1 -->|pre - admission payload UID| C499["respf 499"]
203 + D1 -->|queued| C499
204 + D1 -->|running/awaiting| TMR["cancel() + fallback timer"]
205 + D1 -->|unknown/done| NOP["debug no-op"]
206 + TMR -->|timer fires & still unresolved| C499
207 + R501 --> FIN["manager finalizer"]
208 + R503 --> FIN
209 + R500 --> FIN
210 + C499 --> FIN
211 + HRESP["Handler/dyncfg responder terminal output"] --> FIN
212 + FIN --> TF["tryFinalize(): first wins, tombstone set, emit FUNCRESULT"]
213 + TF --> SREL["scheduler.complete(key, uid)"]
214 + SREL -->|promote next same-key request| W
215 + TF --> OUT["stdout FUNCRESULT"]
216 + HRESP -->|late duplicate| DROP["drop + debug log"]
217 + A -->|ctx . Done| F
218 + A -->|input close| F2["Shutdown(bounded canceling path)"]
219 +```
src/go/plugin/framework/functions/ext.go
+33 -10
@@ -2,6 +2,8 @@
2
3 package functions
4
5 +import "strings"
6 +
7 func (m *Manager) Register(name string, fn func(Function)) {
8 if fn == nil {
9 m.Warningf("not registering '%s': nil function", name)
@@ -11,11 +13,11 @@ func (m *Manager) Register(name string, fn func(Function)) {
13 m.mux.Lock()
14 defer m.mux.Unlock()
15
14 - fs, ok := m.FunctionRegistry[name]
16 + fs, ok := m.functionRegistry[name]
17 if !ok {
18 m.Debugf("registering function '%s' (direct)", name)
19 fs = &functionSet{prefixes: make(map[string]func(Function))}
18 - m.FunctionRegistry[name] = fs
20 + m.functionRegistry[name] = fs
21 } else {
22 if fs.direct != nil {
23 m.Warningf("re-registering direct function '%s'", name)
@@ -31,8 +33,8 @@ func (m *Manager) Unregister(name string) {
33 m.mux.Lock()
34 defer m.mux.Unlock()
35
34 - if _, ok := m.FunctionRegistry[name]; ok {
35 - delete(m.FunctionRegistry, name)
36 + if _, ok := m.functionRegistry[name]; ok {
37 + delete(m.functionRegistry, name)
38 m.Debugf("unregistering function '%s'", name)
39 }
40 }
@@ -50,26 +52,47 @@ func (m *Manager) RegisterPrefix(name, prefix string, fn func(Function)) {
52 m.mux.Lock()
53 defer m.mux.Unlock()
54
53 - fs := m.FunctionRegistry[name]
55 + fs := m.functionRegistry[name]
56 if fs == nil {
57 fs = &functionSet{prefixes: make(map[string]func(Function))}
56 - m.FunctionRegistry[name] = fs
58 + m.functionRegistry[name] = fs
59 }
60
61 if _, exists := fs.prefixes[prefix]; exists {
62 m.Warningf("re-registering function '%s' with prefix '%s'", name, prefix)
61 - } else {
62 - m.Debugf("registering function '%s' with prefix '%s'", name, prefix)
63 + fs.prefixes[prefix] = fn
64 + return
65 + }
66 +
67 + for existing := range fs.prefixes {
68 + if prefixesOverlap(existing, prefix) {
69 + m.Errorf(
70 + "not registering function '%s' with prefix '%s': overlaps with existing prefix '%s'",
71 + name,
72 + prefix,
73 + existing,
74 + )
75 + return
76 + }
77 }
78
79 + m.Debugf("registering function '%s' with prefix '%s'", name, prefix)
80 fs.prefixes[prefix] = fn
81 }
82
83 +func prefixesOverlap(a, b string) bool {
84 + if a == "" || b == "" {
85 + return false
86 + }
87 +
88 + return strings.HasPrefix(a, b) || strings.HasPrefix(b, a)
89 +}
90 +
91 func (m *Manager) UnregisterPrefix(name, prefix string) {
92 m.mux.Lock()
93 defer m.mux.Unlock()
94
72 - fs, ok := m.FunctionRegistry[name]
95 + fs, ok := m.functionRegistry[name]
96 if !ok || fs.prefixes == nil {
97 return
98 }
@@ -80,6 +103,6 @@ func (m *Manager) UnregisterPrefix(name, prefix string) {
103 }
104
105 if fs.direct == nil && len(fs.prefixes) == 0 {
83 - delete(m.FunctionRegistry, name)
106 + delete(m.functionRegistry, name)
107 }
108 }
src/go/plugin/framework/functions/finalizer.go new
+16
@@ -0,0 +1,16 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package functions
4 +
5 +// TerminalFinalizer routes terminal response emission for a transaction UID.
6 +type TerminalFinalizer func(uid, source string, emit func()) bool
7 +
8 +// DirectTerminalFinalizer emits a terminal response without manager-level
9 +// deduplication.
10 +func DirectTerminalFinalizer(_ string, _ string, emit func()) bool {
11 + if emit == nil {
12 + return false
13 + }
14 + emit()
15 + return true
16 +}
src/go/plugin/framework/functions/input.go
+3 -2
@@ -9,7 +9,7 @@ import (
9 )
10
11 type input interface {
12 - lines() chan string
12 + lines() <-chan string
13 }
14
15 func newStdinInput() input {
@@ -22,6 +22,7 @@ type stdinReader struct {
22 }
23
24 func (in *stdinReader) run() {
25 + defer close(in.linesCh)
26 sc := bufio.NewScanner(bufio.NewReader(os.Stdin))
27
28 for sc.Scan() {
@@ -29,7 +30,7 @@ func (in *stdinReader) run() {
30 }
31 }
32
32 -func (in *stdinReader) lines() chan string {
33 +func (in *stdinReader) lines() <-chan string {
34 in.once.Do(func() {
35 in.linesCh = make(chan string)
36 go in.run()
src/go/plugin/framework/functions/manager.go
+637 -78
@@ -4,17 +4,20 @@ package functions
4
5 import (
6 "context"
7 - "encoding/json"
7 + "errors"
8 "fmt"
9 "log/slog"
10 "strconv"
11 "strings"
12 "sync"
13 + "sync/atomic"
14 "time"
15
16 "github.com/netdata/netdata/go/plugins/logger"
17 + "github.com/netdata/netdata/go/plugins/pkg/metrix"
18 "github.com/netdata/netdata/go/plugins/pkg/netdataapi"
19 "github.com/netdata/netdata/go/plugins/pkg/safewriter"
20 + "github.com/netdata/netdata/go/plugins/plugin/framework/runtimecomp"
21 )
22
23 type functionSet struct {
@@ -22,15 +25,76 @@ type functionSet struct {
25 prefixes map[string]func(Function) // for prefix-multiplexed names
26 }
27
28 +const (
29 + // defaultWorkerCount stays at 1 for now.
30 + // Working theory: remaining concurrency risk is in collector MethodHandler
31 + // implementations (shared mutable state and lifecycle races), not in manager
32 + // queue/worker logic.
33 + // TODO: establish and document a MethodHandler goroutine-safety contract
34 + // before increasing this default.
35 + defaultWorkerCount = 1
36 + defaultQueueSize = 64
37 + defaultCancelFallbackDelay = 5 * time.Second
38 + defaultShutdownDrainTimeout = 8 * time.Second
39 + defaultTombstoneTTL = 60 * time.Second
40 + defaultAwaitingWarnDelay = 30 * time.Second
41 +)
42 +
43 +type invocationState uint8
44 +
45 +const (
46 + stateQueued invocationState = iota + 1
47 + stateRunning
48 + stateAwaitingResult
49 +)
50 +
51 +type invocationAdmission uint8
52 +
53 +const (
54 + invocationAdmissionAccepted invocationAdmission = iota + 1
55 + invocationAdmissionDuplicateActive
56 + invocationAdmissionDuplicateTombstone
57 + invocationAdmissionInvalid
58 +)
59 +
60 +type invocationRequest struct {
61 + fn *Function
62 + handler func(Function)
63 + ctx context.Context
64 + scheduleKey string
65 +}
66 +
67 +type invocationRecord struct {
68 + state invocationState
69 + cancel context.CancelFunc
70 + cancelRequested bool
71 + fallbackTimer *time.Timer
72 + awaitingTimer *time.Timer
73 + awaitingSince time.Time
74 + scheduleKey string
75 +}
76 +
77 func NewManager() *Manager {
78 + runtimeStore := metrix.NewRuntimeStore()
79 return &Manager{
80 Logger: logger.New().With(
81 slog.String("component", "functions manager"),
82 ),
30 - api: netdataapi.New(safewriter.Stdout),
31 - input: newStdinInput(),
32 - mux: &sync.Mutex{},
33 - FunctionRegistry: make(map[string]*functionSet),
83 + api: netdataapi.New(safewriter.Stdout),
84 + input: newStdinInput(),
85 + mux: &sync.Mutex{},
86 + functionRegistry: make(map[string]*functionSet),
87 + workerCount: defaultWorkerCount,
88 + queueSize: defaultQueueSize,
89 + invStateMux: &sync.Mutex{},
90 + invState: make(map[string]*invocationRecord),
91 + tombstones: make(map[string]time.Time),
92 + tombstoneTTL: defaultTombstoneTTL,
93 + cancelFallbackDelay: defaultCancelFallbackDelay,
94 + shutdownDrainTimeout: defaultShutdownDrainTimeout,
95 + awaitingWarnDelay: defaultAwaitingWarnDelay,
96 + runtimeStore: runtimeStore,
97 + runtimeMetrics: newManagerRuntimeMetrics(runtimeStore),
98 }
99 }
100
@@ -42,149 +106,644 @@ type Manager struct {
106 input input
107
108 mux *sync.Mutex
45 - FunctionRegistry map[string]*functionSet
109 + functionRegistry map[string]*functionSet
110 +
111 + workerCount int
112 + queueSize int
113 +
114 + scheduler *keyScheduler
115 +
116 + invStateMux *sync.Mutex
117 + invState map[string]*invocationRecord
118 + tombstones map[string]time.Time
119 + tombstoneTTL time.Duration
120 + cancelFallbackDelay time.Duration
121 + shutdownDrainTimeout time.Duration
122 + awaitingWarnDelay time.Duration
123 + stopping atomic.Bool
124 +
125 + runtimeService runtimecomp.Service
126 + runtimeStore metrix.RuntimeStore
127 + runtimeMetrics *managerRuntimeMetrics
128 + runtimeComponentName string
129 + runtimeComponentRegistered bool
130 }
131
132 func (m *Manager) Run(ctx context.Context, quitCh chan struct{}) {
133 m.Info("instance is started")
134 defer func() { m.Info("instance is stopped") }()
135
52 - var wg sync.WaitGroup
53 -
54 - wg.Add(1)
55 - go func() { defer wg.Done(); m.run(ctx, quitCh) }()
56 -
57 - wg.Wait()
136 + if err := m.registerRuntimeComponent(); err != nil {
137 + m.Warningf("runtime metrics registration failed: %v", err)
138 + } else {
139 + defer m.unregisterRuntimeComponent()
140 + }
141
59 - <-ctx.Done()
142 + m.run(ctx, quitCh)
143 }
144
145 func (m *Manager) run(ctx context.Context, quitCh chan struct{}) {
146 parser := newInputParser()
147 + m.scheduler = newKeyScheduler(m.queueSize)
148 + m.observeSchedulerPending()
149 + var workersWG sync.WaitGroup
150 +
151 + m.startWorkers(&workersWG)
152
153 for {
154 select {
155 case <-ctx.Done():
156 + m.shutdown(false, true, quitCh, &workersWG)
157 return
158 case line, ok := <-m.input.lines():
159 if !ok {
160 + m.shutdown(false, true, quitCh, &workersWG)
161 return
162 }
73 - if line == "QUIT" {
74 - if quitCh != nil {
75 - quitCh <- struct{}{}
76 - return
77 - }
78 - continue
79 - }
80 -
81 - fn, err := parser.parse(line)
163 + event, err := parser.parseEvent(line)
164 if err != nil {
165 m.Warningf("parse function: %v ('%s')", err, line)
166 continue
167 }
86 - if fn == nil {
87 - continue
88 - }
168
90 - handler, ok := m.lookupFunction(fn.Name)
91 - if !ok {
92 - m.Infof("skipping execution of '%s': unregistered function", fn.Name)
93 - m.respf(fn, 501, "unregistered function: %s", fn.Name)
169 + switch event.kind {
170 + case inputEventNone, inputEventProgress:
171 continue
95 - }
96 - if handler == nil {
97 - m.Warningf("skipping execution of '%s': nil function registered", fn.Name)
98 - m.respf(fn, 501, "nil function: %s", fn.Name)
172 + case inputEventQuit:
173 + m.shutdown(true, true, quitCh, &workersWG)
174 + return
175 + case inputEventCancel:
176 + m.handleCancelEvent(event)
177 continue
178 + case inputEventCall:
179 + m.dispatchInvocation(ctx, event.fn)
180 }
181 + }
182 + }
183 +}
184 +
185 +func (m *Manager) startWorkers(workersWG *sync.WaitGroup) {
186 + if workersWG == nil {
187 + return
188 + }
189 +
190 + for range m.workerCount {
191 + workersWG.Go(m.runWorker)
192 + }
193 +}
194 +
195 +func (m *Manager) shutdown(signalQuit, cancelInflight bool, quitCh chan struct{}, workersWG *sync.WaitGroup) {
196 + m.setStopping(true)
197 + m.signalQuitIfRequested(signalQuit, quitCh)
198 + m.stopSchedulerAdmission()
199 + timedOut := m.waitWorkers(workersWG)
200 + m.finalizeUnresolvedOnShutdown(cancelInflight, timedOut)
201 +}
202 +
203 +func (m *Manager) signalQuitIfRequested(signalQuit bool, quitCh chan struct{}) {
204 + if signalQuit && quitCh != nil {
205 + quitCh <- struct{}{}
206 + }
207 +}
208 +
209 +func (m *Manager) stopSchedulerAdmission() {
210 + if m.scheduler != nil {
211 + m.scheduler.stopAccepting()
212 + }
213 +}
214 +
215 +func (m *Manager) waitWorkers(workersWG *sync.WaitGroup) bool {
216 + if workersWG == nil {
217 + return false
218 + }
219 +
220 + drainCtx, cancelDrain := context.WithTimeout(context.Background(), m.shutdownDrainTimeout)
221 + defer cancelDrain()
222 +
223 + done := make(chan struct{})
224 + go func() {
225 + defer close(done)
226 + workersWG.Wait()
227 + }()
228 +
229 + select {
230 + case <-done:
231 + return false
232 + case <-drainCtx.Done():
233 + return true
234 + }
235 +}
236 +
237 +func (m *Manager) finalizeUnresolvedOnShutdown(cancelInflight, timedOut bool) {
238 + if !cancelInflight {
239 + return
240 + }
241 + if !timedOut && !m.hasActiveInvocations() {
242 + return
243 + }
244 +
245 + m.cancelAllInvocations()
246 + m.forceFinalizeAll(499, "request canceled during shutdown")
247 + if m.scheduler != nil {
248 + m.scheduler.stop()
249 + }
250 +}
251
102 - handler(*fn)
252 +func (m *Manager) dispatchInvocation(parentCtx context.Context, fn *Function) {
253 + if fn == nil {
254 + return
255 + }
256 + if m.isStopping() {
257 + m.respf(fn, 503, "functions manager is stopping")
258 + return
259 + }
260 +
261 + handler, scheduleKey, ok := m.lookupFunctionRoute(*fn)
262 + if !ok {
263 + m.Infof("skipping execution of '%s': unregistered function", fn.Name)
264 + m.respf(fn, 501, "unregistered function: %s", fn.Name)
265 + return
266 + }
267 + if handler == nil {
268 + m.Warningf("skipping execution of '%s': nil function registered", fn.Name)
269 + m.respf(fn, 501, "nil function: %s", fn.Name)
270 + return
271 + }
272 +
273 + reqCtx, cancel := context.WithCancel(parentCtx)
274 + switch m.trySetInvocationState(fn.UID, stateQueued, cancel, scheduleKey) {
275 + case invocationAdmissionAccepted:
276 + // admitted
277 + case invocationAdmissionDuplicateActive:
278 + cancel()
279 + // Do not emit terminal output for duplicates of an active UID. Emitting
280 + // via tryFinalize would mutate active tracking for the original invocation.
281 + m.Warningf("ignoring duplicate active transaction id: %s", fn.UID)
282 + m.observeDuplicateUIDIgnored()
283 + return
284 + case invocationAdmissionDuplicateTombstone:
285 + cancel()
286 + m.Warningf("ignoring duplicate recently finalized transaction id: %s", fn.UID)
287 + m.observeDuplicateUIDIgnored()
288 + return
289 + case invocationAdmissionInvalid:
290 + cancel()
291 + m.Warningf("ignoring invalid transaction id: %q", fn.UID)
292 + return
293 + default:
294 + cancel()
295 + m.Warningf("ignoring transaction id '%s': unsupported admission state", fn.UID)
296 + return
297 + }
298 +
299 + req := &invocationRequest{
300 + fn: fn,
301 + handler: handler,
302 + ctx: reqCtx,
303 + scheduleKey: scheduleKey,
304 + }
305 +
306 + if m.scheduler == nil {
307 + cancel()
308 + m.respf(fn, 503, "functions manager is stopping")
309 + return
310 + }
311 +
312 + if err := m.scheduler.enqueue(req); err != nil {
313 + cancel()
314 + switch {
315 + case errors.Is(err, errSchedulerQueueFull):
316 + m.respf(fn, 503, "function queue is full")
317 + m.observeQueueFull()
318 + case errors.Is(err, errSchedulerStopping):
319 + m.respf(fn, 503, "functions manager is stopping")
320 + case errors.Is(err, errSchedulerInvalid):
321 + m.respf(fn, 500, "invalid scheduler request")
322 + default:
323 + m.respf(fn, 503, "function queue is full")
324 }
325 + return
326 }
327 + m.observeSchedulerPending()
328 }
329
107 -func (m *Manager) lookupFunction(name string) (func(Function), bool) {
330 +func (m *Manager) trySetInvocationState(uid string, state invocationState, cancel context.CancelFunc, scheduleKey string) invocationAdmission {
331 + if uid == "" {
332 + return invocationAdmissionInvalid
333 + }
334 +
335 + m.invStateMux.Lock()
336 + defer m.invStateMux.Unlock()
337 +
338 + m.pruneExpiredTombstonesLocked(time.Now())
339 +
340 + if _, ok := m.tombstones[uid]; ok {
341 + return invocationAdmissionDuplicateTombstone
342 + }
343 +
344 + if _, ok := m.invState[uid]; ok {
345 + return invocationAdmissionDuplicateActive
346 + }
347 + m.invState[uid] = &invocationRecord{
348 + state: state,
349 + cancel: cancel,
350 + scheduleKey: scheduleKey,
351 + }
352 + m.observeInvocationsLocked()
353 + return invocationAdmissionAccepted
354 +}
355 +
356 +func (m *Manager) setAwaitingResultState(uid string, fnTimeout time.Duration) {
357 + if uid == "" {
358 + return
359 + }
360 +
361 + m.invStateMux.Lock()
362 + defer m.invStateMux.Unlock()
363 +
364 + rec, ok := m.invState[uid]
365 + if !ok || rec == nil {
366 + return
367 + }
368 +
369 + rec.state = stateAwaitingResult
370 + rec.awaitingSince = time.Now()
371 + m.startAwaitingTimerLocked(uid, rec, fnTimeout)
372 + m.observeInvocationsLocked()
373 +}
374 +
375 +func (m *Manager) logAwaitingResult(uid string) {
376 + if uid == "" {
377 + return
378 + }
379 +
380 + m.invStateMux.Lock()
381 + rec, ok := m.invState[uid]
382 + if !ok || rec == nil || rec.state != stateAwaitingResult {
383 + m.invStateMux.Unlock()
384 + return
385 + }
386 + age := time.Since(rec.awaitingSince)
387 + m.invStateMux.Unlock()
388 +
389 + m.Warningf("transaction uid '%s' is still awaiting terminal response after %s", uid, age)
390 +}
391 +
392 +func (m *Manager) startInvocation(uid string) bool {
393 + if uid == "" {
394 + return false
395 + }
396 +
397 + m.invStateMux.Lock()
398 + defer m.invStateMux.Unlock()
399 +
400 + rec, ok := m.invState[uid]
401 + if !ok || rec == nil || rec.cancelRequested {
402 + return false
403 + }
404 + rec.state = stateRunning
405 + m.observeInvocationsLocked()
406 + return true
407 +}
408 +
409 +func (m *Manager) handleCancelEvent(event inputEvent) {
410 + uid := event.uid
411 + if uid == "" {
412 + return
413 + }
414 +
415 + if event.preAdmission {
416 + m.respUID(uid, 499, "request canceled")
417 + return
418 + }
419 +
420 + state, ok := m.requestCancellation(uid)
421 + if !ok {
422 + m.Debugf("ignoring cancel for unknown transaction id: %s", uid)
423 + return
424 + }
425 + if state == stateQueued {
426 + m.respUID(uid, 499, "request canceled")
427 + }
428 +}
429 +
430 +func (m *Manager) requestCancellation(uid string) (invocationState, bool) {
431 + m.invStateMux.Lock()
432 + defer m.invStateMux.Unlock()
433 +
434 + rec, ok := m.invState[uid]
435 + if !ok || rec == nil {
436 + return 0, false
437 + }
438 +
439 + if rec.cancelRequested {
440 + return rec.state, true
441 + }
442 + rec.cancelRequested = true
443 + if rec.cancel != nil {
444 + rec.cancel()
445 + }
446 +
447 + if rec.state == stateQueued && m.scheduler != nil {
448 + m.scheduler.cancelQueued(rec.scheduleKey, uid)
449 + m.observeSchedulerPending()
450 + }
451 +
452 + if rec.state == stateRunning || rec.state == stateAwaitingResult {
453 + m.startCancelFallbackTimerLocked(uid, rec)
454 + }
455 +
456 + return rec.state, true
457 +}
458 +
459 +func (m *Manager) setStopping(v bool) {
460 + m.stopping.Store(v)
461 +}
462 +
463 +func (m *Manager) isStopping() bool {
464 + return m.stopping.Load()
465 +}
466 +
467 +func (m *Manager) hasActiveInvocations() bool {
468 + m.invStateMux.Lock()
469 + defer m.invStateMux.Unlock()
470 + return len(m.invState) > 0
471 +}
472 +
473 +// TerminalFinalizer returns the manager-bound terminal finalizer for responder wiring.
474 +func (m *Manager) TerminalFinalizer() TerminalFinalizer {
475 + return m.finalizeTerminal
476 +}
477 +
478 +func (m *Manager) finalizeTerminal(uid, source string, emit func()) bool {
479 + return m.tryFinalize(uid, source, emit)
480 +}
481 +
482 +func (m *Manager) cancelAllInvocations() {
483 + m.invStateMux.Lock()
484 + defer m.invStateMux.Unlock()
485 +
486 + for _, rec := range m.invState {
487 + if rec == nil {
488 + continue
489 + }
490 + rec.cancelRequested = true
491 + if rec.cancel != nil {
492 + rec.cancel()
493 + }
494 + }
495 +}
496 +
497 +func (m *Manager) forceFinalizeAll(code int, message string) {
498 + m.invStateMux.Lock()
499 + uids := make([]string, 0, len(m.invState))
500 + for uid := range m.invState {
501 + uids = append(uids, uid)
502 + }
503 + m.invStateMux.Unlock()
504 +
505 + for _, uid := range uids {
506 + m.respUID(uid, code, "%s", message)
507 + }
508 +}
509 +
510 +// tryFinalize emits a terminal response once per transaction UID.
511 +// Later terminal attempts for the same UID are dropped while tombstone is active.
512 +func (m *Manager) tryFinalize(uid, source string, emit func()) bool {
513 + if uid == "" || emit == nil {
514 + return false
515 + }
516 +
517 + m.invStateMux.Lock()
518 + now := time.Now()
519 + m.pruneExpiredTombstonesLocked(now)
520 + if _, ok := m.tombstones[uid]; ok {
521 + m.invStateMux.Unlock()
522 + m.Debugf("dropping late terminal response for uid '%s' (source=%s)", uid, source)
523 + m.observeLateTerminalDropped()
524 + return false
525 + }
526 +
527 + var scheduleKey string
528 + if rec, ok := m.invState[uid]; ok && rec != nil {
529 + m.stopTimersLocked(rec)
530 + scheduleKey = rec.scheduleKey
531 + }
532 + delete(m.invState, uid)
533 + m.tombstones[uid] = now.Add(m.tombstoneTTL)
534 + m.observeInvocationsLocked()
535 + m.invStateMux.Unlock()
536 +
537 + if scheduleKey != "" && m.scheduler != nil {
538 + m.scheduler.complete(scheduleKey, uid)
539 + m.observeSchedulerPending()
540 + }
541 +
542 + emit()
543 + return true
544 +}
545 +
546 +func (m *Manager) pruneExpiredTombstonesLocked(now time.Time) {
547 + for uid, expiresAt := range m.tombstones {
548 + if !expiresAt.After(now) {
549 + delete(m.tombstones, uid)
550 + }
551 + }
552 +}
553 +
554 +// startAwaitingTimerLocked starts/refreshes awaiting-result warning timer.
555 +// Caller must hold m.invStateMux.
556 +func (m *Manager) startAwaitingTimerLocked(uid string, rec *invocationRecord, fnTimeout time.Duration) {
557 + if uid == "" || rec == nil {
558 + return
559 + }
560 +
561 + delay := m.awaitingWarnDelay
562 + if fnTimeout > 0 && fnTimeout < delay {
563 + delay = fnTimeout
564 + }
565 + if delay <= 0 {
566 + return
567 + }
568 +
569 + if rec.awaitingTimer != nil {
570 + rec.awaitingTimer.Stop()
571 + }
572 + uidCopy := uid
573 + rec.awaitingTimer = time.AfterFunc(delay, func() {
574 + m.logAwaitingResult(uidCopy)
575 + })
576 +}
577 +
578 +// startCancelFallbackTimerLocked starts cancel fallback timer once.
579 +// Caller must hold m.invStateMux.
580 +func (m *Manager) startCancelFallbackTimerLocked(uid string, rec *invocationRecord) {
581 + if uid == "" || rec == nil || rec.fallbackTimer != nil {
582 + return
583 + }
584 +
585 + uidCopy := uid
586 + rec.fallbackTimer = time.AfterFunc(m.cancelFallbackDelay, func() {
587 + m.observeCancelFallback()
588 + m.respUID(uidCopy, 499, "request canceled")
589 + })
590 +}
591 +
592 +// stopTimersLocked stops invocation timers and clears timer references.
593 +// Caller must hold m.invStateMux.
594 +func (m *Manager) stopTimersLocked(rec *invocationRecord) {
595 + if rec == nil {
596 + return
597 + }
598 +
599 + if rec.fallbackTimer != nil {
600 + rec.fallbackTimer.Stop()
601 + rec.fallbackTimer = nil
602 + }
603 + if rec.awaitingTimer != nil {
604 + rec.awaitingTimer.Stop()
605 + rec.awaitingTimer = nil
606 + }
607 +}
608 +
609 +type functionSnapshot struct {
610 + direct func(Function)
611 + prefixes map[string]func(Function)
612 +}
613 +
614 +func (m *Manager) snapshotFunction(name string) (functionSnapshot, bool) {
615 m.mux.Lock()
109 - fs, ok := m.FunctionRegistry[name]
110 - var (
111 - direct func(Function)
112 - prefixes map[string]func(Function)
113 - )
616 + fs, ok := m.functionRegistry[name]
617 + snap := functionSnapshot{}
618 if ok && fs != nil {
115 - direct = fs.direct
619 + snap.direct = fs.direct
620 if len(fs.prefixes) > 0 {
117 - prefixes = make(map[string]func(Function), len(fs.prefixes))
621 + snap.prefixes = make(map[string]func(Function), len(fs.prefixes))
622 for prefix, handler := range fs.prefixes {
119 - prefixes[prefix] = handler
623 + snap.prefixes[prefix] = handler
624 }
625 }
626 }
627 m.mux.Unlock()
628
629 if !ok || fs == nil {
630 + return functionSnapshot{}, false
631 + }
632 + return snap, true
633 +}
634 +
635 +func matchPrefix(prefixes map[string]func(Function), id string) (string, func(Function), bool) {
636 + if len(prefixes) == 0 || id == "" {
637 + return "", nil, false
638 + }
639 +
640 + for prefix, handler := range prefixes {
641 + if strings.HasPrefix(id, prefix) {
642 + return prefix, handler, true
643 + }
644 + }
645 +
646 + return "", nil, false
647 +}
648 +
649 +func (m *Manager) lookupFunctionRoute(fn Function) (handler func(Function), scheduleKey string, ok bool) {
650 + snap, ok := m.snapshotFunction(fn.Name)
651 + if !ok {
652 + return nil, "", false
653 + }
654 + unknownHandler := m.unknownFunctionHandler()
655 +
656 + if len(snap.prefixes) > 0 {
657 + if len(fn.Args) > 0 {
658 + id := fn.Args[0]
659 + if prefix, routeHandler, matched := matchPrefix(snap.prefixes, id); matched && routeHandler != nil {
660 + return routeHandler, routeScheduleKey(fn.Name, prefix), true
661 + }
662 + }
663 +
664 + return unknownHandler, routeScheduleKey(fn.Name, scheduleKeyUnmatched), true
665 + }
666 +
667 + if snap.direct != nil {
668 + return snap.direct, routeScheduleKey(fn.Name, ""), true
669 + }
670 +
671 + return unknownHandler, routeScheduleKey(fn.Name, scheduleKeyDirectMissing), true
672 +}
673 +
674 +// lookupFunction returns a snapshot handler used by existing tests to verify
675 +// registry snapshot semantics at lookup time.
676 +func (m *Manager) lookupFunction(name string) (func(Function), bool) {
677 + snap, ok := m.snapshotFunction(name)
678 + if !ok {
679 return nil, false
680 }
681 + unknownHandler := m.unknownFunctionHandler()
682
683 return func(f Function) {
130 - if len(prefixes) > 0 {
131 - m.handlePrefixRouting(f, prefixes)
684 + if len(snap.prefixes) > 0 {
685 + if len(f.Args) > 0 {
686 + id := f.Args[0]
687 + if _, handler, matched := matchPrefix(snap.prefixes, id); matched && handler != nil {
688 + handler(f)
689 + return
690 + }
691 + }
692 + unknownHandler(f)
693 return
694 }
695
135 - if direct != nil {
136 - direct(f)
696 + if snap.direct != nil {
697 + snap.direct(f)
698 return
699 }
700
140 - m.respf(&f, 503, "unknown function '%s' (%v)", f.Name, f.Args)
701 + unknownHandler(f)
702 }, true
703 }
704
144 -func (m *Manager) handlePrefixRouting(f Function, prefixes map[string]func(Function)) {
145 - if len(f.Args) == 0 {
705 +func (m *Manager) unknownFunctionHandler() func(Function) {
706 + return func(f Function) {
707 m.respf(&f, 503, "unknown function '%s' (%v)", f.Name, f.Args)
147 - return
708 }
709 +}
710
150 - id := f.Args[0]
151 - for prefix, handler := range prefixes {
152 - if strings.HasPrefix(id, prefix) {
153 - handler(f)
154 - return
155 - }
711 +const (
712 + scheduleKeyUnmatched = "__unmatched__"
713 + scheduleKeyDirectMissing = "__direct_missing__"
714 +)
715 +
716 +func routeScheduleKey(name, discriminator string) string {
717 + if discriminator == "" {
718 + return name
719 }
720 + return name + "|" + discriminator
721 +}
722
158 - m.respf(&f, 503, "unknown function '%s' (%v)", f.Name, f.Args)
723 +func (m *Manager) respUID(uid string, code int, msgf string, a ...any) {
724 + if uid == "" {
725 + return
726 + }
727 + m.respf(&Function{UID: uid}, code, msgf, a...)
728 }
729
730 func (m *Manager) respf(fn *Function, code int, msgf string, a ...any) {
162 - msg := fmt.Sprintf(msgf, a...)
163 -
164 - var bs []byte
165 - if code >= 400 && code < 600 {
166 - bs, _ = json.Marshal(struct {
167 - Status int `json:"status"`
168 - ErrorMessage string `json:"errorMessage"`
169 - }{
170 - Status: code,
171 - ErrorMessage: msg,
172 - })
173 - } else {
174 - bs, _ = json.Marshal(struct {
175 - Status int `json:"status"`
176 - Message string `json:"message"`
177 - }{
178 - Status: code,
179 - Message: msg,
180 - })
731 + if fn == nil || fn.UID == "" {
732 + return
733 }
734
183 - m.api.FUNCRESULT(netdataapi.FunctionResult{
735 + msg := fmt.Sprintf(msgf, a...)
736 + bs := BuildJSONPayload(code, msg)
737 +
738 + res := netdataapi.FunctionResult{
739 UID: fn.UID,
740 ContentType: "application/json",
741 Payload: string(bs),
742 Code: strconv.Itoa(code),
743 ExpireTimestamp: strconv.FormatInt(time.Now().Unix(), 10),
744 + }
745 +
746 + m.finalizeTerminal(fn.UID, "functions.manager.respf", func() {
747 + m.api.FUNCRESULT(res)
748 })
749 }
src/go/plugin/framework/functions/manager_flow_test.go new
+742
@@ -0,0 +1,742 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package functions
4 +
5 +import (
6 + "bytes"
7 + "context"
8 + "fmt"
9 + "strings"
10 + "sync"
11 + "sync/atomic"
12 + "testing"
13 + "time"
14 +
15 + "github.com/netdata/netdata/go/plugins/pkg/netdataapi"
16 + "github.com/stretchr/testify/assert"
17 + "github.com/stretchr/testify/require"
18 +)
19 +
20 +const (
21 + testPermissions = "0xFFFF"
22 + testSource = "method=api,role=test"
23 +)
24 +
25 +type chanInput struct {
26 + ch chan string
27 +}
28 +
29 +func (m *chanInput) lines() <-chan string {
30 + return m.ch
31 +}
32 +
33 +type safeBuffer struct {
34 + mu sync.Mutex
35 + b bytes.Buffer
36 +}
37 +
38 +func (s *safeBuffer) Write(p []byte) (int, error) {
39 + s.mu.Lock()
40 + defer s.mu.Unlock()
41 + return s.b.Write(p)
42 +}
43 +
44 +func (s *safeBuffer) String() string {
45 + s.mu.Lock()
46 + defer s.mu.Unlock()
47 + return s.b.String()
48 +}
49 +
50 +func newFlowManager() (*Manager, *safeBuffer) {
51 + mgr := NewManager()
52 + buf := &safeBuffer{}
53 + mgr.api = netdataapi.New(buf)
54 + return mgr, buf
55 +}
56 +
57 +func functionLine(uid, name string) string {
58 + return fmt.Sprintf(`FUNCTION %s 10 "%s" %s "%s"`, uid, name, testPermissions, testSource)
59 +}
60 +
61 +func payloadStartCmd(uid, name string) string {
62 + return fmt.Sprintf(`FUNCTION_PAYLOAD %s 10 "%s" %s "%s" application/json`, uid, name, testPermissions, testSource)
63 +}
64 +
65 +func waitForSubstring(t *testing.T, f func() string, substr string, timeout time.Duration) {
66 + t.Helper()
67 +
68 + deadline := time.Now().Add(timeout)
69 + for time.Now().Before(deadline) {
70 + if strings.Contains(f(), substr) {
71 + return
72 + }
73 + time.Sleep(10 * time.Millisecond)
74 + }
75 + t.Fatalf("timeout waiting for substring %q in output: %s", substr, f())
76 +}
77 +
78 +func startFlowManager(t *testing.T, mgr *Manager) (context.CancelFunc, chan struct{}) {
79 + t.Helper()
80 +
81 + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
82 + done := make(chan struct{})
83 + go func() {
84 + defer close(done)
85 + mgr.Run(ctx, nil)
86 + }()
87 + return cancel, done
88 +}
89 +
90 +func waitForDone(t *testing.T, done <-chan struct{}) {
91 + t.Helper()
92 +
93 + select {
94 + case <-done:
95 + case <-time.After(3 * time.Second):
96 + t.Fatal("timeout waiting for manager run to complete")
97 + }
98 +}
99 +
100 +func waitForCondition(t *testing.T, timeout time.Duration, fn func() bool, desc string) {
101 + t.Helper()
102 +
103 + deadline := time.Now().Add(timeout)
104 + for time.Now().Before(deadline) {
105 + if fn() {
106 + return
107 + }
108 + time.Sleep(10 * time.Millisecond)
109 + }
110 + t.Fatalf("timeout waiting for condition: %s", desc)
111 +}
112 +
113 +func TestManager_FlowScenarios(t *testing.T) {
114 + tests := map[string]struct {
115 + run func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer)
116 + }{
117 + "queued cancel emits 499 and skips execution": {
118 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
119 + var (
120 + mu sync.Mutex
121 + executed []string
122 + )
123 + started := make(chan struct{}, 1)
124 + release := make(chan struct{})
125 +
126 + mgr.Register("fn", func(fn Function) {
127 + mu.Lock()
128 + executed = append(executed, fn.UID)
129 + mu.Unlock()
130 + if fn.UID == "tx1" {
131 + started <- struct{}{}
132 + <-release
133 + }
134 + })
135 +
136 + cancel, done := startFlowManager(t, mgr)
137 + defer cancel()
138 +
139 + in.ch <- functionLine("tx1", "fn")
140 + <-started
141 + in.ch <- functionLine("tx2", "fn")
142 + in.ch <- "FUNCTION_CANCEL tx2"
143 + close(release)
144 + close(in.ch)
145 + waitForDone(t, done)
146 +
147 + waitForSubstring(t, out.String, "FUNCTION_RESULT_BEGIN tx2 499", time.Second)
148 + mu.Lock()
149 + defer mu.Unlock()
150 + assert.Equal(t, []string{"tx1"}, executed)
151 + },
152 + },
153 + "running cancel fallback emits 499 once": {
154 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
155 + mgr.cancelFallbackDelay = 50 * time.Millisecond
156 + started := make(chan struct{}, 1)
157 + release := make(chan struct{})
158 +
159 + mgr.Register("fn", func(fn Function) {
160 + if fn.UID == "tx1" {
161 + started <- struct{}{}
162 + <-release
163 + }
164 + })
165 +
166 + cancel, done := startFlowManager(t, mgr)
167 + defer cancel()
168 +
169 + in.ch <- functionLine("tx1", "fn")
170 + <-started
171 + in.ch <- "FUNCTION_CANCEL tx1"
172 + waitForSubstring(t, out.String, "FUNCTION_RESULT_BEGIN tx1 499", time.Second)
173 +
174 + close(release)
175 + close(in.ch)
176 + waitForDone(t, done)
177 +
178 + assert.Equal(t, 1, strings.Count(out.String(), "FUNCTION_RESULT_BEGIN tx1 499"))
179 + },
180 + },
181 + "repeated cancel for same uid still emits one terminal response": {
182 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
183 + mgr.cancelFallbackDelay = 50 * time.Millisecond
184 + started := make(chan struct{}, 1)
185 + release := make(chan struct{})
186 +
187 + mgr.Register("fn", func(fn Function) {
188 + if fn.UID == "tx1" {
189 + started <- struct{}{}
190 + <-release
191 + }
192 + })
193 +
194 + cancel, done := startFlowManager(t, mgr)
195 + defer cancel()
196 +
197 + in.ch <- functionLine("tx1", "fn")
198 + <-started
199 + in.ch <- "FUNCTION_CANCEL tx1"
200 + in.ch <- "FUNCTION_CANCEL tx1"
201 + waitForSubstring(t, out.String, "FUNCTION_RESULT_BEGIN tx1 499", time.Second)
202 +
203 + close(release)
204 + close(in.ch)
205 + waitForDone(t, done)
206 +
207 + assert.Equal(t, 1, strings.Count(out.String(), "FUNCTION_RESULT_BEGIN tx1 499"))
208 + },
209 + },
210 + "running cancel drops late terminal response": {
211 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
212 + mgr.cancelFallbackDelay = 50 * time.Millisecond
213 + started := make(chan struct{}, 1)
214 + release := make(chan struct{})
215 +
216 + mgr.Register("fn", func(fn Function) {
217 + if fn.UID == "tx1" {
218 + started <- struct{}{}
219 + <-release
220 + mgr.respUID(fn.UID, 200, "late response")
221 + }
222 + })
223 +
224 + cancel, done := startFlowManager(t, mgr)
225 + defer cancel()
226 +
227 + in.ch <- functionLine("tx1", "fn")
228 + <-started
229 + in.ch <- "FUNCTION_CANCEL tx1"
230 + waitForSubstring(t, out.String, "FUNCTION_RESULT_BEGIN tx1 499", time.Second)
231 +
232 + close(release)
233 + close(in.ch)
234 + waitForDone(t, done)
235 +
236 + got := out.String()
237 + assert.Equal(t, 1, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 499"))
238 + assert.Equal(t, 0, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 200"))
239 + },
240 + },
241 + "payload pre-admission cancel emits 499 and skips handler": {
242 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
243 + var calls atomic.Int32
244 + mgr.Register("fn", func(Function) { calls.Add(1) })
245 +
246 + cancel, done := startFlowManager(t, mgr)
247 + defer cancel()
248 +
249 + in.ch <- payloadStartCmd("tx1", "fn")
250 + in.ch <- "payload line"
251 + in.ch <- "FUNCTION_CANCEL tx1"
252 + close(in.ch)
253 + waitForDone(t, done)
254 +
255 + waitForSubstring(t, out.String, "FUNCTION_RESULT_BEGIN tx1 499", time.Second)
256 + assert.EqualValues(t, 0, calls.Load())
257 + },
258 + },
259 + "duplicate active uid is ignored without corrupting lane progression": {
260 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
261 + mgr.workerCount = 2
262 + started := make(chan struct{}, 1)
263 + release := make(chan struct{})
264 + var (
265 + calls atomic.Int32
266 + running atomic.Int32
267 + maxSeen atomic.Int32
268 + )
269 +
270 + mgr.Register("fn", func(fn Function) {
271 + calls.Add(1)
272 +
273 + curr := running.Add(1)
274 + for {
275 + prev := maxSeen.Load()
276 + if curr <= prev || maxSeen.CompareAndSwap(prev, curr) {
277 + break
278 + }
279 + }
280 + defer running.Add(-1)
281 +
282 + if fn.UID == "tx1" {
283 + started <- struct{}{}
284 + <-release
285 + }
286 + mgr.respUID(fn.UID, 200, "ok")
287 + })
288 +
289 + cancel, done := startFlowManager(t, mgr)
290 + defer cancel()
291 +
292 + in.ch <- functionLine("tx1", "fn")
293 + <-started
294 + // Same-key request should stay queued until tx1 completes.
295 + in.ch <- functionLine("tx2", "fn")
296 + // Duplicate active UID must not advance lanes or finalize tx1.
297 + in.ch <- functionLine("tx1", "fn")
298 + close(release)
299 + close(in.ch)
300 + waitForDone(t, done)
301 +
302 + got := out.String()
303 + assert.Equal(t, 1, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 200"))
304 + assert.Equal(t, 1, strings.Count(got, "FUNCTION_RESULT_BEGIN tx2 200"))
305 + assert.Equal(t, 0, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 409"))
306 + assert.EqualValues(t, 2, calls.Load())
307 + // Same key must remain serialized despite duplicate UID input.
308 + assert.EqualValues(t, 1, maxSeen.Load())
309 + },
310 + },
311 + "duplicate tombstoned uid is ignored without extra terminal output": {
312 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
313 + var calls atomic.Int32
314 +
315 + mgr.Register("fn", func(fn Function) {
316 + calls.Add(1)
317 + mgr.respUID(fn.UID, 200, "ok")
318 + })
319 +
320 + cancel, done := startFlowManager(t, mgr)
321 + defer cancel()
322 +
323 + in.ch <- functionLine("tx1", "fn")
324 + waitForSubstring(t, out.String, "FUNCTION_RESULT_BEGIN tx1 200", time.Second)
325 +
326 + // Re-send same UID while tombstone is still active.
327 + in.ch <- functionLine("tx1", "fn")
328 + close(in.ch)
329 + waitForDone(t, done)
330 +
331 + got := out.String()
332 + assert.Equal(t, 1, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 200"))
333 + assert.Equal(t, 0, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 409"))
334 + assert.EqualValues(t, 1, calls.Load())
335 + },
336 + },
337 + "queue full is rejected with 503": {
338 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
339 + mgr.queueSize = 1
340 + mgr.workerCount = 1
341 + started := make(chan struct{}, 1)
342 + release := make(chan struct{})
343 +
344 + mgr.Register("fn", func(fn Function) {
345 + if fn.UID == "tx1" {
346 + started <- struct{}{}
347 + <-release
348 + }
349 + mgr.respUID(fn.UID, 200, "ok")
350 + })
351 +
352 + cancel, done := startFlowManager(t, mgr)
353 + defer cancel()
354 +
355 + in.ch <- functionLine("tx1", "fn")
356 + <-started
357 + in.ch <- functionLine("tx2", "fn")
358 + in.ch <- functionLine("tx3", "fn")
359 + waitForSubstring(t, out.String, "FUNCTION_RESULT_BEGIN tx3 503", time.Second)
360 +
361 + close(release)
362 + close(in.ch)
363 + waitForDone(t, done)
364 + },
365 + },
366 + "panic in handler emits 500": {
367 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
368 + mgr.Register("fn", func(Function) { panic("boom") })
369 +
370 + cancel, done := startFlowManager(t, mgr)
371 + defer cancel()
372 +
373 + in.ch <- functionLine("tx1", "fn")
374 + close(in.ch)
375 + waitForDone(t, done)
376 +
377 + waitForSubstring(t, out.String, "FUNCTION_RESULT_BEGIN tx1 500", time.Second)
378 + },
379 + },
380 + "cancel for unknown uid is no-op": {
381 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
382 + cancel, done := startFlowManager(t, mgr)
383 + defer cancel()
384 +
385 + in.ch <- "FUNCTION_CANCEL unknown"
386 + close(in.ch)
387 + waitForDone(t, done)
388 +
389 + assert.Equal(t, 0, strings.Count(out.String(), "FUNCTION_RESULT_BEGIN"))
390 + },
391 + },
392 + "cancel after completion is no-op": {
393 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
394 + mgr.Register("fn", func(fn Function) {
395 + mgr.respUID(fn.UID, 200, "done")
396 + })
397 +
398 + cancel, done := startFlowManager(t, mgr)
399 + defer cancel()
400 +
401 + in.ch <- functionLine("tx1", "fn")
402 + waitForSubstring(t, out.String, "FUNCTION_RESULT_BEGIN tx1 200", time.Second)
403 + in.ch <- "FUNCTION_CANCEL tx1"
404 + close(in.ch)
405 + waitForDone(t, done)
406 +
407 + got := out.String()
408 + assert.Equal(t, 1, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 200"))
409 + assert.Equal(t, 0, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 499"))
410 + },
411 + },
412 + "stdin close uses canceling shutdown and force-finalizes unresolved requests": {
413 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
414 + mgr.shutdownDrainTimeout = 50 * time.Millisecond
415 + started := make(chan struct{}, 1)
416 + block := make(chan struct{})
417 +
418 + mgr.Register("fn", func(fn Function) {
419 + started <- struct{}{}
420 + <-block
421 + mgr.respUID(fn.UID, 200, "late")
422 + })
423 +
424 + cancel, done := startFlowManager(t, mgr)
425 + defer cancel()
426 +
427 + in.ch <- functionLine("tx1", "fn")
428 + <-started
429 + close(in.ch)
430 + waitForDone(t, done)
431 +
432 + got := out.String()
433 + assert.Equal(t, 1, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 499"))
434 + assert.Equal(t, 0, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 200"))
435 + },
436 + },
437 + "late terminal output after shutdown is dropped by tombstone guard": {
438 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
439 + mgr.shutdownDrainTimeout = 50 * time.Millisecond
440 + started := make(chan struct{}, 1)
441 + block := make(chan struct{})
442 + doneResp := make(chan struct{})
443 +
444 + mgr.Register("fn", func(fn Function) {
445 + started <- struct{}{}
446 + <-block
447 + mgr.respUID(fn.UID, 200, "late")
448 + close(doneResp)
449 + })
450 +
451 + cancel, done := startFlowManager(t, mgr)
452 + defer cancel()
453 +
454 + in.ch <- functionLine("tx1", "fn")
455 + <-started
456 + close(in.ch)
457 + waitForDone(t, done)
458 +
459 + // Release handler after manager has already force-finalized and returned.
460 + close(block)
461 + select {
462 + case <-doneResp:
463 + case <-time.After(time.Second):
464 + t.Fatal("timed out waiting for late handler response")
465 + }
466 +
467 + got := out.String()
468 + assert.Equal(t, 1, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 499"))
469 + assert.Equal(t, 0, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 200"))
470 + },
471 + },
472 + "shutdown finalizes unresolved awaiting_result even when workers are drained": {
473 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
474 + mgr.shutdownDrainTimeout = 250 * time.Millisecond
475 + mgr.awaitingWarnDelay = time.Second
476 + returned := make(chan struct{}, 1)
477 +
478 + mgr.Register("fn", func(Function) {
479 + // Return without terminal response.
480 + returned <- struct{}{}
481 + })
482 +
483 + cancel, done := startFlowManager(t, mgr)
484 + defer cancel()
485 +
486 + in.ch <- functionLine("tx1", "fn")
487 + <-returned
488 + waitForCondition(t, time.Second, func() bool {
489 + mgr.invStateMux.Lock()
490 + defer mgr.invStateMux.Unlock()
491 + rec, ok := mgr.invState["tx1"]
492 + return ok && rec != nil && rec.state == stateAwaitingResult
493 + }, "tx1 reaches awaiting_result before shutdown")
494 +
495 + close(in.ch)
496 + waitForDone(t, done)
497 +
498 + got := out.String()
499 + assert.Equal(t, 1, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 499"))
500 + assert.Equal(t, 0, strings.Count(got, "FUNCTION_RESULT_BEGIN tx1 200"))
501 + },
502 + },
503 + }
504 +
505 + for name, tc := range tests {
506 + t.Run(name, func(t *testing.T) {
507 + mgr, out := newFlowManager()
508 + in := &chanInput{ch: make(chan string, 16)}
509 + mgr.input = in
510 + tc.run(t, mgr, in, out)
511 + })
512 + }
513 +}
514 +
515 +func TestManager_InvocationStateScenarios(t *testing.T) {
516 + tests := map[string]struct {
517 + run func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer)
518 + }{
519 + "worker return transitions to awaiting_result until terminal output": {
520 + run: func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer) {
521 + mgr.workerCount = 1
522 + returned := make(chan struct{}, 1)
523 + releaseResponse := make(chan struct{})
524 +
525 + mgr.Register("fn", func(fn Function) {
526 + go func() {
527 + <-releaseResponse
528 + mgr.respUID(fn.UID, 200, "ok")
529 + }()
530 + returned <- struct{}{}
531 + })
532 +
533 + cancel, done := startFlowManager(t, mgr)
534 + defer cancel()
535 +
536 + in.ch <- functionLine("tx1", "fn")
537 + <-returned
538 +
539 + waitForCondition(t, time.Second, func() bool {
540 + mgr.invStateMux.Lock()
541 + defer mgr.invStateMux.Unlock()
542 + rec, ok := mgr.invState["tx1"]
543 + return ok && rec != nil && rec.state == stateAwaitingResult
544 + }, "transaction tx1 reaches awaiting_result state")
545 +
546 + close(releaseResponse)
547 + waitForSubstring(t, out.String, "FUNCTION_RESULT_BEGIN tx1 200", time.Second)
548 +
549 + waitForCondition(t, time.Second, func() bool {
550 + mgr.invStateMux.Lock()
551 + defer mgr.invStateMux.Unlock()
552 + _, ok := mgr.invState["tx1"]
553 + return !ok
554 + }, "transaction tx1 is removed from active state after finalization")
555 +
556 + close(in.ch)
557 + waitForDone(t, done)
558 + },
559 + },
560 + }
561 +
562 + for name, tc := range tests {
563 + t.Run(name, func(t *testing.T) {
564 + mgr, out := newFlowManager()
565 + in := &chanInput{ch: make(chan string, 16)}
566 + mgr.input = in
567 + tc.run(t, mgr, in, out)
568 + })
569 + }
570 +}
571 +
572 +func TestManager_tryFinalize(t *testing.T) {
573 + tests := map[string]struct {
574 + run func(t *testing.T, mgr *Manager)
575 + }{
576 + "first finalization wins and tombstone blocks immediate reuse": {
577 + run: func(t *testing.T, mgr *Manager) {
578 + calls := 0
579 + ok := mgr.tryFinalize("uid1", "test.first", func() { calls++ })
580 + require.True(t, ok)
581 + assert.Equal(t, 1, calls)
582 +
583 + ok = mgr.tryFinalize("uid1", "test.late", func() { calls++ })
584 + require.False(t, ok)
585 + assert.Equal(t, 1, calls)
586 +
587 + admitted := mgr.trySetInvocationState("uid1", stateQueued, func() {}, "fn")
588 + assert.Equal(t, invocationAdmissionDuplicateTombstone, admitted)
589 +
590 + mgr.invStateMux.Lock()
591 + mgr.tombstones["uid1"] = time.Now().Add(-time.Second)
592 + mgr.invStateMux.Unlock()
593 +
594 + admitted = mgr.trySetInvocationState("uid1", stateQueued, func() {}, "fn")
595 + assert.Equal(t, invocationAdmissionAccepted, admitted)
596 + },
597 + },
598 + "empty uid or nil emitter is rejected": {
599 + run: func(t *testing.T, mgr *Manager) {
600 + assert.False(t, mgr.tryFinalize("", "source", func() {}))
601 + assert.False(t, mgr.tryFinalize("uid1", "source", nil))
602 + },
603 + },
604 + }
605 +
606 + for name, tc := range tests {
607 + t.Run(name, func(t *testing.T) {
608 + tc.run(t, NewManager())
609 + })
610 + }
611 +}
612 +
613 +func TestManager_WorkerPoolConcurrencyBound(t *testing.T) {
614 + tests := map[string]struct {
615 + workerCount int
616 + input []string
617 + register func(t *testing.T, mgr *Manager, current, maxSeen *atomic.Int32)
618 + assertions func(t *testing.T, maxSeen int32)
619 + }{
620 + "same key is serialized": {
621 + workerCount: 4,
622 + input: []string{
623 + functionLine("tx-1", "fn"),
624 + functionLine("tx-2", "fn"),
625 + functionLine("tx-3", "fn"),
626 + functionLine("tx-4", "fn"),
627 + },
628 + register: func(t *testing.T, mgr *Manager, current, maxSeen *atomic.Int32) {
629 + t.Helper()
630 + mgr.Register("fn", func(fn Function) {
631 + c := current.Add(1)
632 + for {
633 + prev := maxSeen.Load()
634 + if c <= prev || maxSeen.CompareAndSwap(prev, c) {
635 + break
636 + }
637 + }
638 +
639 + time.Sleep(30 * time.Millisecond)
640 + current.Add(-1)
641 + mgr.respUID(fn.UID, 200, "ok")
642 + })
643 + },
644 + assertions: func(t *testing.T, maxSeen int32) {
645 + t.Helper()
646 + assert.Equal(t, int32(1), maxSeen)
647 + },
648 + },
649 + "different keys run concurrently": {
650 + workerCount: 4,
651 + input: []string{
652 + functionLine("tx-a1", "fnA"),
653 + functionLine("tx-b1", "fnB"),
654 + functionLine("tx-a2", "fnA"),
655 + functionLine("tx-b2", "fnB"),
656 + },
657 + register: func(t *testing.T, mgr *Manager, current, maxSeen *atomic.Int32) {
658 + t.Helper()
659 + registerFn := func(name string) {
660 + mgr.Register(name, func(fn Function) {
661 + c := current.Add(1)
662 + for {
663 + prev := maxSeen.Load()
664 + if c <= prev || maxSeen.CompareAndSwap(prev, c) {
665 + break
666 + }
667 + }
668 +
669 + time.Sleep(40 * time.Millisecond)
670 + current.Add(-1)
671 + mgr.respUID(fn.UID, 200, "ok")
672 + })
673 + }
674 + registerFn("fnA")
675 + registerFn("fnB")
676 + },
677 + assertions: func(t *testing.T, maxSeen int32) {
678 + t.Helper()
679 + assert.GreaterOrEqual(t, maxSeen, int32(2))
680 + assert.LessOrEqual(t, maxSeen, int32(4))
681 + },
682 + },
683 + }
684 +
685 + for name, tc := range tests {
686 + t.Run(name, func(t *testing.T) {
687 + mgr, out := newFlowManager()
688 + mgr.workerCount = tc.workerCount
689 + mgr.queueSize = len(tc.input) + tc.workerCount
690 + in := &chanInput{ch: make(chan string, len(tc.input)+tc.workerCount)}
691 + mgr.input = in
692 +
693 + var current atomic.Int32
694 + var maxSeen atomic.Int32
695 +
696 + tc.register(t, mgr, &current, &maxSeen)
697 +
698 + cancel, done := startFlowManager(t, mgr)
699 + defer cancel()
700 +
701 + for _, line := range tc.input {
702 + in.ch <- line
703 + }
704 + close(in.ch)
705 + waitForDone(t, done)
706 +
707 + got := out.String()
708 + assert.Equal(t, len(tc.input), strings.Count(got, "FUNCTION_RESULT_BEGIN tx-"))
709 + tc.assertions(t, maxSeen.Load())
710 + })
711 + }
712 +}
713 +
714 +func TestManager_DispatchInvocationStopping(t *testing.T) {
715 + tests := map[string]struct {
716 + run func(t *testing.T, mgr *Manager, out *safeBuffer)
717 + }{
718 + "stopping manager rejects dispatch without tracking state": {
719 + run: func(t *testing.T, mgr *Manager, out *safeBuffer) {
720 + mgr.Register("fn", func(Function) { t.Fatal("handler should not execute when manager is stopping") })
721 + mgr.setStopping(true)
722 +
723 + fn := &Function{UID: "tx-stop", Name: "fn"}
724 + mgr.dispatchInvocation(context.Background(), fn)
725 +
726 + waitForSubstring(t, out.String, "FUNCTION_RESULT_BEGIN tx-stop 503", time.Second)
727 +
728 + mgr.invStateMux.Lock()
729 + _, ok := mgr.invState["tx-stop"]
730 + mgr.invStateMux.Unlock()
731 + assert.False(t, ok)
732 + },
733 + },
734 + }
735 +
736 + for name, tc := range tests {
737 + t.Run(name, func(t *testing.T) {
738 + mgr, out := newFlowManager()
739 + tc.run(t, mgr, out)
740 + })
741 + }
742 +}
src/go/plugin/framework/functions/manager_snapshot_test.go
+59 -26
@@ -5,43 +5,76 @@ package functions
5 import (
6 "testing"
7
8 + "github.com/stretchr/testify/assert"
9 "github.com/stretchr/testify/require"
10 )
11
11 -func TestLookupFunction_UsesDirectSnapshot(t *testing.T) {
12 - mgr := NewManager()
12 +func TestLookupFunction_SnapshotScenarios(t *testing.T) {
13 + tests := map[string]struct {
14 + run func(t *testing.T, mgr *Manager)
15 + }{
16 + "uses direct snapshot": {
17 + run: func(t *testing.T, mgr *Manager) {
18 + called := make(chan struct{}, 1)
19 + mgr.Register("fn", func(Function) { called <- struct{}{} })
20
14 - called := make(chan struct{}, 1)
15 - mgr.Register("fn", func(Function) { called <- struct{}{} })
21 + handler, ok := mgr.lookupFunction("fn")
22 + require.True(t, ok)
23
17 - handler, ok := mgr.lookupFunction("fn")
18 - require.True(t, ok)
24 + mgr.Unregister("fn")
25 + handler(Function{Name: "fn"})
26
20 - mgr.Unregister("fn")
21 - handler(Function{Name: "fn"})
27 + select {
28 + case <-called:
29 + default:
30 + t.Fatal("snapshot handler should still invoke the originally resolved direct function")
31 + }
32 + },
33 + },
34 + "uses prefix snapshot": {
35 + run: func(t *testing.T, mgr *Manager) {
36 + called := make(chan struct{}, 1)
37 + mgr.RegisterPrefix("config", "collector:", func(Function) { called <- struct{}{} })
38
23 - select {
24 - case <-called:
25 - default:
26 - t.Fatal("snapshot handler should still invoke the originally resolved direct function")
27 - }
28 -}
39 + handler, ok := mgr.lookupFunction("config")
40 + require.True(t, ok)
41 +
42 + mgr.UnregisterPrefix("config", "collector:")
43 + handler(Function{Name: "config", Args: []string{"collector:job"}})
44
30 -func TestLookupFunction_UsesPrefixSnapshot(t *testing.T) {
31 - mgr := NewManager()
45 + select {
46 + case <-called:
47 + default:
48 + t.Fatal("snapshot handler should still route using the prefix set captured at lookup time")
49 + }
50 + },
51 + },
52 + "overlapping prefix registration is rejected": {
53 + run: func(t *testing.T, mgr *Manager) {
54 + longHits := 0
55 + shortHits := 0
56
33 - called := make(chan struct{}, 1)
34 - mgr.RegisterPrefix("config", "collector:", func(Function) { called <- struct{}{} })
57 + mgr.RegisterPrefix("config", "collector:", func(Function) { shortHits++ })
58 + mgr.RegisterPrefix("config", "collector:job:", func(Function) { longHits++ })
59
36 - handler, ok := mgr.lookupFunction("config")
37 - require.True(t, ok)
60 + require.NotNil(t, mgr.functionRegistry["config"])
61 + require.Len(t, mgr.functionRegistry["config"].prefixes, 1)
62 + _, longRegistered := mgr.functionRegistry["config"].prefixes["collector:job:"]
63 + assert.False(t, longRegistered)
64
39 - mgr.UnregisterPrefix("config", "collector:")
40 - handler(Function{Name: "config", Args: []string{"collector:job"}})
65 + handler, ok := mgr.lookupFunction("config")
66 + require.True(t, ok)
67 +
68 + handler(Function{Name: "config", Args: []string{"collector:job:alpha"}})
69 + assert.Equal(t, 0, longHits)
70 + assert.Equal(t, 1, shortHits)
71 + },
72 + },
73 + }
74
42 - select {
43 - case <-called:
44 - default:
45 - t.Fatal("snapshot handler should still route using the prefix set captured at lookup time")
75 + for name, tc := range tests {
76 + t.Run(name, func(t *testing.T) {
77 + tc.run(t, NewManager())
78 + })
79 }
80 }
src/go/plugin/framework/functions/manager_test.go
+117 -77
@@ -8,17 +8,23 @@ import (
8 "fmt"
9 "sort"
10 "strings"
11 + "sync"
12 "testing"
13 "time"
14
15 "github.com/stretchr/testify/assert"
16 )
17
18 +const (
19 + managerTestPermissions = "0xFFFF"
20 + managerTestSource = "method=api,role=test"
21 +)
22 +
23 func TestNewManager(t *testing.T) {
24 mgr := NewManager()
25
26 assert.NotNilf(t, mgr.input, "Input")
21 - assert.NotNilf(t, mgr.FunctionRegistry, "FunctionRegistry")
27 + assert.NotNilf(t, mgr.functionRegistry, "FunctionRegistry")
28 }
29
30 func TestManager_Register(t *testing.T) {
@@ -67,7 +73,7 @@ func TestManager_Register(t *testing.T) {
73 }
74
75 var got []string
70 - for name := range mgr.FunctionRegistry {
76 + for name := range mgr.functionRegistry {
77 got = append(got, name)
78 }
79 sort.Strings(got)
@@ -118,6 +124,20 @@ func TestManager_RegisterPrefix(t *testing.T) {
124 },
125 expected: []string{"config:collector:"},
126 },
127 + "overlapping prefix is rejected (short first)": {
128 + input: []inputFn{
129 + {name: "config", prefix: "collector:"},
130 + {name: "config", prefix: "collector:job:"},
131 + },
132 + expected: []string{"config:collector:"},
133 + },
134 + "overlapping prefix is rejected (long first)": {
135 + input: []inputFn{
136 + {name: "config", prefix: "collector:job:"},
137 + {name: "config", prefix: "collector:"},
138 + },
139 + expected: []string{"config:collector:job:"},
140 + },
141 }
142
143 for name, test := range tests {
@@ -133,7 +153,7 @@ func TestManager_RegisterPrefix(t *testing.T) {
153 }
154
155 var got []string
136 - for fname, fs := range mgr.FunctionRegistry {
156 + for fname, fs := range mgr.functionRegistry {
157 if fs == nil || len(fs.prefixes) == 0 {
158 continue
159 }
@@ -218,7 +238,7 @@ func TestManager_UnregisterPrefix(t *testing.T) {
238 }
239
240 var got []string
221 - for fname, fs := range mgr.FunctionRegistry {
241 + for fname, fs := range mgr.functionRegistry {
242 if fs == nil || len(fs.prefixes) == 0 {
243 continue
244 }
@@ -242,9 +262,9 @@ func TestManager_Run(t *testing.T) {
262 }{
263 "valid function: single": {
264 register: []string{"fn1"},
245 - input: `
246 -FUNCTION UID 1 "fn1 arg1 arg2" 0xFFFF "method=api,role=test"
247 -`,
265 + input: fmt.Sprintf(`
266 +FUNCTION UID 1 "fn1 arg1 arg2" %s "%s"
267 +`, managerTestPermissions, managerTestSource),
268 expected: []Function{
269 {
270 key: lineFunction,
@@ -252,8 +272,8 @@ FUNCTION UID 1 "fn1 arg1 arg2" 0xFFFF "method=api,role=test"
272 Timeout: time.Second,
273 Name: "fn1",
274 Args: []string{"arg1", "arg2"},
255 - Permissions: "0xFFFF",
256 - Source: "method=api,role=test",
275 + Permissions: managerTestPermissions,
276 + Source: managerTestSource,
277 ContentType: "",
278 Payload: nil,
279 },
@@ -261,30 +281,30 @@ FUNCTION UID 1 "fn1 arg1 arg2" 0xFFFF "method=api,role=test"
281 },
282 "valid function: multiple": {
283 register: []string{"fn1", "fn2"},
264 - input: `
265 -FUNCTION UID 1 "fn1 arg1 arg2" 0xFFFF "method=api,role=test"
266 -FUNCTION UID 1 "fn2 arg1 arg2" 0xFFFF "method=api,role=test"
267 -`,
284 + input: fmt.Sprintf(`
285 +FUNCTION UID1 1 "fn1 arg1 arg2" %s "%s"
286 +FUNCTION UID2 1 "fn2 arg1 arg2" %s "%s"
287 +`, managerTestPermissions, managerTestSource, managerTestPermissions, managerTestSource),
288 expected: []Function{
289 {
290 key: lineFunction,
271 - UID: "UID",
291 + UID: "UID1",
292 Timeout: time.Second,
293 Name: "fn1",
294 Args: []string{"arg1", "arg2"},
275 - Permissions: "0xFFFF",
276 - Source: "method=api,role=test",
295 + Permissions: managerTestPermissions,
296 + Source: managerTestSource,
297 ContentType: "",
298 Payload: nil,
299 },
300 {
301 key: lineFunction,
282 - UID: "UID",
302 + UID: "UID2",
303 Timeout: time.Second,
304 Name: "fn2",
305 Args: []string{"arg1", "arg2"},
286 - Permissions: "0xFFFF",
287 - Source: "method=api,role=test",
306 + Permissions: managerTestPermissions,
307 + Source: managerTestSource,
308 ContentType: "",
309 Payload: nil,
310 },
@@ -292,12 +312,12 @@ FUNCTION UID 1 "fn2 arg1 arg2" 0xFFFF "method=api,role=test"
312 },
313 "valid function: single with payload": {
314 register: []string{"fn1", "fn2"},
295 - input: `
296 -FUNCTION_PAYLOAD UID 1 "fn1 arg1 arg2" 0xFFFF "method=api,role=test" application/json
315 + input: fmt.Sprintf(`
316 +FUNCTION_PAYLOAD UID 1 "fn1 arg1 arg2" %s "%s" application/json
317 payload line1
318 payload line2
319 FUNCTION_PAYLOAD_END
300 -`,
320 +`, managerTestPermissions, managerTestSource),
321 expected: []Function{
322 {
323 key: lineFunctionPayload,
@@ -305,8 +325,8 @@ FUNCTION_PAYLOAD_END
325 Timeout: time.Second,
326 Name: "fn1",
327 Args: []string{"arg1", "arg2"},
308 - Permissions: "0xFFFF",
309 - Source: "method=api,role=test",
328 + Permissions: managerTestPermissions,
329 + Source: managerTestSource,
330 ContentType: "application/json",
331 Payload: []byte("payload line1\npayload line2"),
332 },
@@ -314,37 +334,37 @@ FUNCTION_PAYLOAD_END
334 },
335 "valid function: multiple with payload": {
336 register: []string{"fn1", "fn2"},
317 - input: `
318 -FUNCTION_PAYLOAD UID 1 "fn1 arg1 arg2" 0xFFFF "method=api,role=test" application/json
337 + input: fmt.Sprintf(`
338 +FUNCTION_PAYLOAD UID1 1 "fn1 arg1 arg2" %s "%s" application/json
339 payload line1
340 payload line2
341 FUNCTION_PAYLOAD_END
342
323 -FUNCTION_PAYLOAD UID 1 "fn2 arg1 arg2" 0xFFFF "method=api,role=test" application/json
343 +FUNCTION_PAYLOAD UID2 1 "fn2 arg1 arg2" %s "%s" application/json
344 payload line3
345 payload line4
346 FUNCTION_PAYLOAD_END
327 -`,
347 +`, managerTestPermissions, managerTestSource, managerTestPermissions, managerTestSource),
348 expected: []Function{
349 {
350 key: lineFunctionPayload,
331 - UID: "UID",
351 + UID: "UID1",
352 Timeout: time.Second,
353 Name: "fn1",
354 Args: []string{"arg1", "arg2"},
335 - Permissions: "0xFFFF",
336 - Source: "method=api,role=test",
355 + Permissions: managerTestPermissions,
356 + Source: managerTestSource,
357 ContentType: "application/json",
358 Payload: []byte("payload line1\npayload line2"),
359 },
360 {
361 key: lineFunctionPayload,
342 - UID: "UID",
362 + UID: "UID2",
363 Timeout: time.Second,
364 Name: "fn2",
365 Args: []string{"arg1", "arg2"},
346 - Permissions: "0xFFFF",
347 - Source: "method=api,role=test",
366 + Permissions: managerTestPermissions,
367 + Source: managerTestSource,
368 ContentType: "application/json",
369 Payload: []byte("payload line3\npayload line4"),
370 },
@@ -352,62 +372,65 @@ FUNCTION_PAYLOAD_END
372 },
373 "valid function: multiple with and without payload": {
374 register: []string{"fn1", "fn2", "fn3", "fn4"},
355 - input: `
356 -FUNCTION_PAYLOAD UID 1 "fn1 arg1 arg2" 0xFFFF "method=api,role=test" application/json
375 + input: fmt.Sprintf(`
376 +FUNCTION_PAYLOAD UID1 1 "fn1 arg1 arg2" %s "%s" application/json
377 payload line1
378 payload line2
379 FUNCTION_PAYLOAD_END
380
361 -FUNCTION UID 1 "fn2 arg1 arg2" 0xFFFF "method=api,role=test"
362 -FUNCTION UID 1 "fn3 arg1 arg2" 0xFFFF "method=api,role=test"
381 +FUNCTION UID2 1 "fn2 arg1 arg2" %s "%s"
382 +FUNCTION UID3 1 "fn3 arg1 arg2" %s "%s"
383
364 -FUNCTION_PAYLOAD UID 1 "fn4 arg1 arg2" 0xFFFF "method=api,role=test" application/json
384 +FUNCTION_PAYLOAD UID4 1 "fn4 arg1 arg2" %s "%s" application/json
385 payload line3
386 payload line4
387 FUNCTION_PAYLOAD_END
368 -`,
388 +`, managerTestPermissions, managerTestSource,
389 + managerTestPermissions, managerTestSource,
390 + managerTestPermissions, managerTestSource,
391 + managerTestPermissions, managerTestSource),
392 expected: []Function{
393 {
394 key: lineFunctionPayload,
372 - UID: "UID",
395 + UID: "UID1",
396 Timeout: time.Second,
397 Name: "fn1",
398 Args: []string{"arg1", "arg2"},
376 - Permissions: "0xFFFF",
377 - Source: "method=api,role=test",
399 + Permissions: managerTestPermissions,
400 + Source: managerTestSource,
401 ContentType: "application/json",
402 Payload: []byte("payload line1\npayload line2"),
403 },
404 {
405 key: lineFunction,
383 - UID: "UID",
406 + UID: "UID2",
407 Timeout: time.Second,
408 Name: "fn2",
409 Args: []string{"arg1", "arg2"},
387 - Permissions: "0xFFFF",
388 - Source: "method=api,role=test",
410 + Permissions: managerTestPermissions,
411 + Source: managerTestSource,
412 ContentType: "",
413 Payload: nil,
414 },
415 {
416 key: lineFunction,
394 - UID: "UID",
417 + UID: "UID3",
418 Timeout: time.Second,
419 Name: "fn3",
420 Args: []string{"arg1", "arg2"},
398 - Permissions: "0xFFFF",
399 - Source: "method=api,role=test",
421 + Permissions: managerTestPermissions,
422 + Source: managerTestSource,
423 ContentType: "",
424 Payload: nil,
425 },
426 {
427 key: lineFunctionPayload,
405 - UID: "UID",
428 + UID: "UID4",
429 Timeout: time.Second,
430 Name: "fn4",
431 Args: []string{"arg1", "arg2"},
409 - Permissions: "0xFFFF",
410 - Source: "method=api,role=test",
432 + Permissions: managerTestPermissions,
433 + Source: managerTestSource,
434 ContentType: "application/json",
435 Payload: []byte("payload line3\npayload line4"),
436 },
@@ -416,46 +439,63 @@ FUNCTION_PAYLOAD_END
439 }
440
441 for name, test := range tests {
419 - t.Run(name, func(t *testing.T) {
420 - mgr := NewManager()
421 -
422 - mgr.input = newMockInput(test.input)
423 -
424 - mock := &mockFunctionExecutor{}
425 - for _, v := range test.register {
426 - mgr.Register(v, mock.execute)
427 - }
442 + for workerProfile, workerCount := range map[string]int{
443 + "single-worker": 1,
444 + "multi-worker": 4,
445 + } {
446 + t.Run(name+"/"+workerProfile, func(t *testing.T) {
447 + mgr := NewManager()
448 + mgr.workerCount = workerCount
449 +
450 + mgr.input = newMockInput(test.input)
451 +
452 + mock := &mockFunctionExecutor{}
453 + for _, v := range test.register {
454 + mgr.Register(v, mock.execute)
455 + }
456
429 - testTime := time.Second * 5
430 - ctx, cancel := context.WithTimeout(context.Background(), testTime)
431 - defer cancel()
457 + testTime := time.Second * 5
458 + ctx, cancel := context.WithTimeout(context.Background(), testTime)
459 + defer cancel()
460
433 - done := make(chan struct{})
461 + done := make(chan struct{})
462
435 - go func() { defer close(done); mgr.Run(ctx, nil) }()
463 + go func() { defer close(done); mgr.Run(ctx, nil) }()
464
437 - timeout := testTime + time.Second*2
438 - tk := time.NewTimer(timeout)
439 - defer tk.Stop()
465 + timeout := testTime + time.Second*2
466 + tk := time.NewTimer(timeout)
467 + defer tk.Stop()
468
441 - select {
442 - case <-done:
443 - assert.Equal(t, test.expected, mock.executed)
444 - case <-tk.C:
445 - t.Errorf("timed out after %s", timeout)
446 - }
447 - })
469 + select {
470 + case <-done:
471 + assert.ElementsMatch(t, test.expected, mock.snapshot())
472 + case <-tk.C:
473 + t.Errorf("timed out after %s", timeout)
474 + }
475 + })
476 + }
477 }
478 }
479
480 type mockFunctionExecutor struct {
481 + mu sync.Mutex
482 executed []Function
483 }
484
485 func (m *mockFunctionExecutor) execute(fn Function) {
486 + m.mu.Lock()
487 + defer m.mu.Unlock()
488 m.executed = append(m.executed, fn)
489 }
490
491 +func (m *mockFunctionExecutor) snapshot() []Function {
492 + m.mu.Lock()
493 + defer m.mu.Unlock()
494 + out := make([]Function, len(m.executed))
495 + copy(out, m.executed)
496 + return out
497 +}
498 +
499 func newMockInput(data string) *mockInput {
500 m := &mockInput{chLines: make(chan string)}
501 sc := bufio.NewScanner(strings.NewReader(data))
@@ -472,6 +512,6 @@ type mockInput struct {
512 chLines chan string
513 }
514
475 -func (m *mockInput) lines() chan string {
515 +func (m *mockInput) lines() <-chan string {
516 return m.chLines
517 }
src/go/plugin/framework/functions/manager_worker.go new
+47
@@ -0,0 +1,47 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package functions
4 +
5 +import "runtime/debug"
6 +
7 +func (m *Manager) runWorker() {
8 + for {
9 + if m.scheduler == nil {
10 + return
11 + }
12 + req, ok := m.scheduler.next()
13 + m.observeSchedulerPending()
14 + if !ok {
15 + return
16 + }
17 + if req == nil || req.fn == nil || req.handler == nil {
18 + continue
19 + }
20 + // Safe to skip: cancel/finalization path calls tryFinalize(), which in turn
21 + // advances per-key lanes via scheduler.complete().
22 + if req.ctx != nil && req.ctx.Err() != nil {
23 + continue
24 + }
25 + if !m.startInvocation(req.fn.UID) {
26 + continue
27 + }
28 +
29 + panicked := false
30 + func() {
31 + defer func() {
32 + if v := recover(); v != nil {
33 + m.Errorf("function handler panic (uid=%s): %v\n%s", req.fn.UID, v, string(debug.Stack()))
34 + panicked = true
35 + }
36 + }()
37 + req.handler(*req.fn)
38 + }()
39 +
40 + if panicked {
41 + m.respUID(req.fn.UID, 500, "function handler panic")
42 + continue
43 + }
44 +
45 + m.setAwaitingResultState(req.fn.UID, req.fn.Timeout)
46 + }
47 +}
src/go/plugin/framework/functions/parser.go
+103 -13
@@ -16,6 +16,9 @@ const (
16 lineFunction = "FUNCTION"
17 lineFunctionPayload = "FUNCTION_PAYLOAD"
18 lineFunctionPayloadEnd = "FUNCTION_PAYLOAD_END"
19 + lineFunctionCancel = "FUNCTION_CANCEL"
20 + lineFunctionProgress = "FUNCTION_PROGRESS"
21 + lineQuit = "QUIT"
22 )
23
24 type Function struct {
@@ -46,8 +49,36 @@ type inputParser struct {
49 }
50
51 func (p *inputParser) parse(line string) (*Function, error) {
52 + event, err := p.parseEvent(line)
53 + if err != nil {
54 + return nil, err
55 + }
56 + if event.kind == inputEventCall {
57 + return event.fn, nil
58 + }
59 + return nil, nil
60 +}
61 +
62 +type inputEventKind uint8
63 +
64 +const (
65 + inputEventNone inputEventKind = iota
66 + inputEventCall
67 + inputEventCancel
68 + inputEventProgress
69 + inputEventQuit
70 +)
71 +
72 +type inputEvent struct {
73 + kind inputEventKind
74 + fn *Function
75 + uid string
76 + preAdmission bool
77 +}
78 +
79 +func (p *inputParser) parseEvent(line string) (inputEvent, error) {
80 if line = strings.TrimSpace(line); line == "" {
50 - return nil, nil
81 + return inputEvent{}, nil
82 }
83
84 if p.readingPayload {
@@ -55,36 +86,68 @@ func (p *inputParser) parse(line string) (*Function, error) {
86 }
87
88 switch {
89 + case line == lineQuit:
90 + return inputEvent{kind: inputEventQuit}, nil
91 + case hasLinePrefix(line, lineFunctionCancel):
92 + return parseCancelEvent(line)
93 + case hasLinePrefix(line, lineFunctionProgress):
94 + return parseProgressEvent(line), nil
95 case strings.HasPrefix(line, lineFunction+" "):
59 - return p.parseFunction(line)
96 + fn, err := p.parseFunction(line)
97 + if err != nil {
98 + return inputEvent{}, err
99 + }
100 + return inputEvent{kind: inputEventCall, fn: fn}, nil
101 case strings.HasPrefix(line, lineFunctionPayload+" "):
102 fn, err := p.parseFunction(line)
103 if err != nil {
63 - return nil, err
104 + return inputEvent{}, err
105 }
106 p.readingPayload = true
107 p.currentFn = fn
108 p.payloadBuf.Reset()
68 - return nil, nil
109 + return inputEvent{}, nil
110 default:
70 - return nil, errors.New("unexpected line format")
111 + return inputEvent{}, errors.New("unexpected line format")
112 }
113 }
114
74 -func (p *inputParser) handlePayloadLine(line string) (*Function, error) {
115 +func (p *inputParser) handlePayloadLine(line string) (inputEvent, error) {
116 if line == lineFunctionPayloadEnd {
117 p.readingPayload = false
118 p.currentFn.Payload = []byte(p.payloadBuf.String())
119 fn := p.currentFn
120 p.currentFn = nil
80 - return fn, nil
121 + p.payloadBuf.Reset()
122 + return inputEvent{kind: inputEventCall, fn: fn}, nil
123 }
124
83 - if strings.HasPrefix(line, lineFunction) {
84 - p.readingPayload = false
85 - p.currentFn = nil
86 - p.payloadBuf.Reset()
87 - return p.parse(line)
125 + if hasLinePrefix(line, lineFunctionCancel) {
126 + event, err := parseCancelEvent(line)
127 + if err != nil {
128 + // Malformed cancel must not affect payload parser state.
129 + return inputEvent{}, err
130 + }
131 +
132 + if p.currentFn != nil && event.uid == p.currentFn.UID {
133 + p.resetPayloadState()
134 + event.preAdmission = true
135 + }
136 + return event, nil
137 + }
138 +
139 + if hasLinePrefix(line, lineFunctionProgress) {
140 + return parseProgressEvent(line), nil
141 + }
142 +
143 + if line == lineQuit {
144 + p.resetPayloadState()
145 + return inputEvent{kind: inputEventQuit}, nil
146 + }
147 +
148 + if hasLinePrefix(line, lineFunction) || strings.HasPrefix(line, lineFunction+"_") {
149 + p.resetPayloadState()
150 + return p.parseEvent(line)
151 }
152
153 if p.payloadBuf.Len() > 0 {
@@ -92,7 +155,34 @@ func (p *inputParser) handlePayloadLine(line string) (*Function, error) {
155 }
156 p.payloadBuf.WriteString(line)
157
95 - return nil, nil
158 + return inputEvent{}, nil
159 +}
160 +
161 +func (p *inputParser) resetPayloadState() {
162 + p.readingPayload = false
163 + p.currentFn = nil
164 + p.payloadBuf.Reset()
165 +}
166 +
167 +func hasLinePrefix(line, keyword string) bool {
168 + return line == keyword || strings.HasPrefix(line, keyword+" ")
169 +}
170 +
171 +func parseCancelEvent(line string) (inputEvent, error) {
172 + parts := strings.Fields(line)
173 + if len(parts) != 2 || parts[0] != lineFunctionCancel || parts[1] == "" {
174 + return inputEvent{}, errors.New("unexpected FUNCTION_CANCEL format")
175 + }
176 + return inputEvent{kind: inputEventCancel, uid: parts[1]}, nil
177 +}
178 +
179 +func parseProgressEvent(line string) inputEvent {
180 + parts := strings.Fields(line)
181 + event := inputEvent{kind: inputEventProgress}
182 + if len(parts) >= 2 {
183 + event.uid = parts[1]
184 + }
185 + return event
186 }
187
188 func (p *inputParser) parseFunction(line string) (*Function, error) {
src/go/plugin/framework/functions/parser_test.go new
+216
@@ -0,0 +1,216 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package functions
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/stretchr/testify/assert"
9 + "github.com/stretchr/testify/require"
10 +)
11 +
12 +const (
13 + parserTestPermissions = "0xFFFF"
14 + parserTestSource = "method=api,role=test"
15 + testPayloadStartLine = `FUNCTION_PAYLOAD tx1 1 "fn1 arg1" ` + parserTestPermissions + ` "` + parserTestSource + `" application/json`
16 + testFunctionLine = `FUNCTION tx2 1 "fn2 arg1" ` + parserTestPermissions + ` "` + parserTestSource + `"`
17 +)
18 +
19 +func TestInputParser_ParseEvent(t *testing.T) {
20 + tests := map[string]struct {
21 + lines []string
22 + wantErr bool
23 + assertEvent func(t *testing.T, events []inputEvent)
24 + assertState func(t *testing.T, p *inputParser)
25 + }{
26 + "cancel in normal mode": {
27 + lines: []string{"FUNCTION_CANCEL tx1"},
28 + assertEvent: func(t *testing.T, events []inputEvent) {
29 + require.Len(t, events, 1)
30 + assert.Equal(t, inputEventCancel, events[0].kind)
31 + assert.Equal(t, "tx1", events[0].uid)
32 + assert.False(t, events[0].preAdmission)
33 + },
34 + },
35 + "progress in normal mode": {
36 + lines: []string{"FUNCTION_PROGRESS tx1 10 100"},
37 + assertEvent: func(t *testing.T, events []inputEvent) {
38 + require.Len(t, events, 1)
39 + assert.Equal(t, inputEventProgress, events[0].kind)
40 + assert.Equal(t, "tx1", events[0].uid)
41 + },
42 + },
43 + "malformed cancel with extra token": {
44 + lines: []string{"FUNCTION_CANCEL tx1 extra"},
45 + wantErr: true,
46 + },
47 + "malformed cancel with missing uid": {
48 + lines: []string{"FUNCTION_CANCEL"},
49 + wantErr: true,
50 + },
51 + "payload cancel with different uid keeps payload": {
52 + lines: []string{testPayloadStartLine, "line1", "FUNCTION_CANCEL tx-other", "line2", "FUNCTION_PAYLOAD_END"},
53 + assertEvent: func(t *testing.T, events []inputEvent) {
54 + require.Len(t, events, 2)
55 + assert.Equal(t, inputEventCancel, events[0].kind)
56 + assert.Equal(t, "tx-other", events[0].uid)
57 + assert.False(t, events[0].preAdmission)
58 +
59 + assert.Equal(t, inputEventCall, events[1].kind)
60 + require.NotNil(t, events[1].fn)
61 + assert.Equal(t, "tx1", events[1].fn.UID)
62 + assert.Equal(t, []byte("line1\nline2"), events[1].fn.Payload)
63 + },
64 + assertState: func(t *testing.T, p *inputParser) {
65 + assert.False(t, p.readingPayload)
66 + assert.Nil(t, p.currentFn)
67 + assert.Equal(t, 0, p.payloadBuf.Len())
68 + },
69 + },
70 + "payload cancel with same uid is pre-admission": {
71 + lines: []string{testPayloadStartLine, "line1", "FUNCTION_CANCEL tx1"},
72 + assertEvent: func(t *testing.T, events []inputEvent) {
73 + require.Len(t, events, 1)
74 + assert.Equal(t, inputEventCancel, events[0].kind)
75 + assert.Equal(t, "tx1", events[0].uid)
76 + assert.True(t, events[0].preAdmission)
77 + },
78 + assertState: func(t *testing.T, p *inputParser) {
79 + assert.False(t, p.readingPayload)
80 + assert.Nil(t, p.currentFn)
81 + assert.Equal(t, 0, p.payloadBuf.Len())
82 + },
83 + },
84 + "malformed cancel during payload keeps parser state": {
85 + lines: []string{testPayloadStartLine, "line1", "FUNCTION_CANCEL tx1 extra"},
86 + wantErr: true,
87 + assertState: func(t *testing.T, p *inputParser) {
88 + assert.True(t, p.readingPayload)
89 + require.NotNil(t, p.currentFn)
90 + assert.Equal(t, "tx1", p.currentFn.UID)
91 + assert.Equal(t, "line1", p.payloadBuf.String())
92 + },
93 + },
94 + "progress during payload keeps payload": {
95 + lines: []string{testPayloadStartLine, "line1", "FUNCTION_PROGRESS tx1 10 100", "line2", "FUNCTION_PAYLOAD_END"},
96 + assertEvent: func(t *testing.T, events []inputEvent) {
97 + require.Len(t, events, 2)
98 + assert.Equal(t, inputEventProgress, events[0].kind)
99 + assert.Equal(t, "tx1", events[0].uid)
100 +
101 + assert.Equal(t, inputEventCall, events[1].kind)
102 + require.NotNil(t, events[1].fn)
103 + assert.Equal(t, []byte("line1\nline2"), events[1].fn.Payload)
104 + },
105 + },
106 + "payload data line with FUNCTION prefix text is preserved": {
107 + lines: []string{testPayloadStartLine, "line1", "FUNCTIONALITY=true", "line2", "FUNCTION_PAYLOAD_END"},
108 + assertEvent: func(t *testing.T, events []inputEvent) {
109 + require.Len(t, events, 1)
110 + assert.Equal(t, inputEventCall, events[0].kind)
111 + require.NotNil(t, events[0].fn)
112 + assert.Equal(t, []byte("line1\nFUNCTIONALITY=true\nline2"), events[0].fn.Payload)
113 + },
114 + },
115 + "unexpected control line during payload aborts partial payload": {
116 + lines: []string{testPayloadStartLine, "line1", testFunctionLine},
117 + assertEvent: func(t *testing.T, events []inputEvent) {
118 + require.Len(t, events, 1)
119 + assert.Equal(t, inputEventCall, events[0].kind)
120 + require.NotNil(t, events[0].fn)
121 + assert.Equal(t, "tx2", events[0].fn.UID)
122 + assert.Nil(t, events[0].fn.Payload)
123 + },
124 + assertState: func(t *testing.T, p *inputParser) {
125 + assert.False(t, p.readingPayload)
126 + assert.Nil(t, p.currentFn)
127 + },
128 + },
129 + "quit during payload aborts payload and emits quit": {
130 + lines: []string{testPayloadStartLine, "line1", "QUIT"},
131 + assertEvent: func(t *testing.T, events []inputEvent) {
132 + require.Len(t, events, 1)
133 + assert.Equal(t, inputEventQuit, events[0].kind)
134 + },
135 + assertState: func(t *testing.T, p *inputParser) {
136 + assert.False(t, p.readingPayload)
137 + assert.Nil(t, p.currentFn)
138 + assert.Equal(t, 0, p.payloadBuf.Len())
139 + },
140 + },
141 + "unknown FUNCTION_ control during payload aborts payload and errors": {
142 + lines: []string{testPayloadStartLine, "line1", "FUNCTION_UNKNOWN tx1"},
143 + wantErr: true,
144 + assertState: func(t *testing.T, p *inputParser) {
145 + assert.False(t, p.readingPayload)
146 + assert.Nil(t, p.currentFn)
147 + assert.Equal(t, 0, p.payloadBuf.Len())
148 + },
149 + },
150 + }
151 +
152 + for name, tc := range tests {
153 + t.Run(name, func(t *testing.T) {
154 + p := newInputParser()
155 + events := make([]inputEvent, 0, len(tc.lines))
156 + var parseErr error
157 +
158 + for _, line := range tc.lines {
159 + ev, err := p.parseEvent(line)
160 + if err != nil {
161 + parseErr = err
162 + break
163 + }
164 + if ev.kind != inputEventNone {
165 + events = append(events, ev)
166 + }
167 + }
168 +
169 + if tc.wantErr {
170 + require.Error(t, parseErr)
171 + } else {
172 + require.NoError(t, parseErr)
173 + }
174 + if !tc.wantErr && tc.assertEvent != nil {
175 + tc.assertEvent(t, events)
176 + }
177 + if tc.assertState != nil {
178 + tc.assertState(t, p)
179 + }
180 + })
181 + }
182 +}
183 +
184 +func TestInputParser_Parse_Wrapper(t *testing.T) {
185 + tests := map[string]struct {
186 + line string
187 + wantFn bool
188 + wantID string
189 + }{
190 + "function line returns function": {
191 + line: `FUNCTION tx1 1 "fn1 arg1" ` + parserTestPermissions + ` "` + parserTestSource + `"`,
192 + wantFn: true,
193 + wantID: "tx1",
194 + },
195 + "cancel line returns nil function": {
196 + line: "FUNCTION_CANCEL tx1",
197 + },
198 + "progress line returns nil function": {
199 + line: "FUNCTION_PROGRESS tx1 10 100",
200 + },
201 + }
202 +
203 + for name, tc := range tests {
204 + t.Run(name, func(t *testing.T) {
205 + p := newInputParser()
206 + fn, err := p.parse(tc.line)
207 + require.NoError(t, err)
208 + if tc.wantFn {
209 + require.NotNil(t, fn)
210 + assert.Equal(t, tc.wantID, fn.UID)
211 + return
212 + }
213 + assert.Nil(t, fn)
214 + })
215 + }
216 +}
src/go/plugin/framework/functions/response_payload.go new
+28
@@ -0,0 +1,28 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package functions
4 +
5 +import "encoding/json"
6 +
7 +// BuildJSONPayload builds the standard JSON payload used by function terminal responses.
8 +func BuildJSONPayload(code int, message string) []byte {
9 + if code >= 400 && code < 600 {
10 + bs, _ := json.Marshal(struct {
11 + Status int `json:"status"`
12 + ErrorMessage string `json:"errorMessage"`
13 + }{
14 + Status: code,
15 + ErrorMessage: message,
16 + })
17 + return bs
18 + }
19 +
20 + bs, _ := json.Marshal(struct {
21 + Status int `json:"status"`
22 + Message string `json:"message"`
23 + }{
24 + Status: code,
25 + Message: message,
26 + })
27 + return bs
28 +}
src/go/plugin/framework/functions/response_payload_test.go new
+56
@@ -0,0 +1,56 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package functions
4 +
5 +import (
6 + "encoding/json"
7 + "testing"
8 +
9 + "github.com/stretchr/testify/assert"
10 + "github.com/stretchr/testify/require"
11 +)
12 +
13 +func TestBuildJSONPayload(t *testing.T) {
14 + tests := map[string]struct {
15 + code int
16 + message string
17 + expectMsgKey string
18 + expectErrorKey string
19 + }{
20 + "success payload uses message key": {
21 + code: 200,
22 + message: "ok",
23 + expectMsgKey: "ok",
24 + },
25 + "error payload uses errorMessage key": {
26 + code: 499,
27 + message: "request canceled",
28 + expectErrorKey: "request canceled",
29 + },
30 + }
31 +
32 + for name, tc := range tests {
33 + t.Run(name, func(t *testing.T) {
34 + payload := BuildJSONPayload(tc.code, tc.message)
35 + require.NotEmpty(t, payload)
36 +
37 + var decoded map[string]any
38 + require.NoError(t, json.Unmarshal(payload, &decoded))
39 +
40 + status, ok := decoded["status"].(float64)
41 + require.True(t, ok)
42 + assert.Equal(t, float64(tc.code), status)
43 +
44 + if tc.expectMsgKey != "" {
45 + assert.Equal(t, tc.expectMsgKey, decoded["message"])
46 + _, hasErrorMessage := decoded["errorMessage"]
47 + assert.False(t, hasErrorMessage)
48 + }
49 + if tc.expectErrorKey != "" {
50 + assert.Equal(t, tc.expectErrorKey, decoded["errorMessage"])
51 + _, hasMessage := decoded["message"]
52 + assert.False(t, hasMessage)
53 + }
54 + })
55 + }
56 +}
src/go/plugin/framework/functions/runtime_component.go new
+62
@@ -0,0 +1,62 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package functions
4 +
5 +import (
6 + "fmt"
7 + "strings"
8 +
9 + "github.com/netdata/netdata/go/plugins/plugin/framework/runtimecomp"
10 +)
11 +
12 +const functionsRuntimeComponentName = "functions.manager"
13 +
14 +func (m *Manager) SetRuntimeService(service runtimecomp.Service) {
15 + if m == nil {
16 + return
17 + }
18 + m.runtimeService = service
19 +}
20 +
21 +func (m *Manager) registerRuntimeComponent() error {
22 + if m == nil || m.runtimeService == nil || m.runtimeComponentRegistered {
23 + return nil
24 + }
25 + if m.runtimeStore == nil {
26 + return fmt.Errorf("nil runtime store")
27 + }
28 +
29 + componentName := strings.TrimSpace(m.runtimeComponentName)
30 + if componentName == "" {
31 + componentName = functionsRuntimeComponentName
32 + }
33 +
34 + cfg := runtimecomp.ComponentConfig{
35 + Name: componentName,
36 + Store: m.runtimeStore,
37 + UpdateEvery: 1,
38 + Autogen: runtimecomp.AutogenPolicy{
39 + Enabled: true,
40 + },
41 + Module: "functions",
42 + JobName: "manager",
43 + JobLabels: map[string]string{
44 + "component": "functions_manager",
45 + },
46 + }
47 + if err := m.runtimeService.RegisterComponent(cfg); err != nil {
48 + return err
49 + }
50 +
51 + m.runtimeComponentName = componentName
52 + m.runtimeComponentRegistered = true
53 + return nil
54 +}
55 +
56 +func (m *Manager) unregisterRuntimeComponent() {
57 + if m == nil || m.runtimeService == nil || !m.runtimeComponentRegistered {
58 + return
59 + }
60 + m.runtimeService.UnregisterComponent(m.runtimeComponentName)
61 + m.runtimeComponentRegistered = false
62 +}
src/go/plugin/framework/functions/runtime_metrics.go new
+128
@@ -0,0 +1,128 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package functions
4 +
5 +import "github.com/netdata/netdata/go/plugins/pkg/metrix"
6 +
7 +const functionsRuntimeMetricPrefix = "netdata.go.plugin.framework.functions.manager"
8 +
9 +type managerRuntimeMetrics struct {
10 + invocationsActive metrix.StatefulGauge
11 + invocationsAwaitingResult metrix.StatefulGauge
12 + schedulerPending metrix.StatefulGauge
13 +
14 + queueFullTotal metrix.StatefulCounter
15 + cancelFallbackTotal metrix.StatefulCounter
16 + lateTerminalDropped metrix.StatefulCounter
17 + duplicateUIDIgnored metrix.StatefulCounter
18 +}
19 +
20 +func newManagerRuntimeMetrics(store metrix.RuntimeStore) *managerRuntimeMetrics {
21 + if store == nil {
22 + return nil
23 + }
24 +
25 + meter := store.Write().StatefulMeter(functionsRuntimeMetricPrefix)
26 + metrics := &managerRuntimeMetrics{
27 + invocationsActive: meter.Gauge(
28 + "invocations_active",
29 + metrix.WithDescription("Current number of active function invocations tracked by UID"),
30 + metrix.WithChartFamily("Framework/Functions/Invocations"),
31 + metrix.WithUnit("invocations"),
32 + ),
33 + invocationsAwaitingResult: meter.Gauge(
34 + "invocations_awaiting_result",
35 + metrix.WithDescription("Current number of active invocations waiting for terminal response"),
36 + metrix.WithChartFamily("Framework/Functions/Invocations"),
37 + metrix.WithUnit("invocations"),
38 + ),
39 + schedulerPending: meter.Gauge(
40 + "scheduler_pending",
41 + metrix.WithDescription("Current number of invocations pending in scheduler"),
42 + metrix.WithChartFamily("Framework/Functions/Scheduler"),
43 + metrix.WithUnit("invocations"),
44 + ),
45 + queueFullTotal: meter.Counter(
46 + "queue_full_total",
47 + metrix.WithDescription("Total number of function requests rejected due to queue full"),
48 + metrix.WithChartFamily("Framework/Functions/Failures"),
49 + metrix.WithUnit("requests"),
50 + ),
51 + cancelFallbackTotal: meter.Counter(
52 + "cancel_fallback_total",
53 + metrix.WithDescription("Total number of function requests finalized by cancel fallback timer"),
54 + metrix.WithChartFamily("Framework/Functions/Cancellation"),
55 + metrix.WithUnit("requests"),
56 + ),
57 + lateTerminalDropped: meter.Counter(
58 + "late_terminal_dropped_total",
59 + metrix.WithDescription("Total number of late terminal responses dropped by tombstone guard"),
60 + metrix.WithChartFamily("Framework/Functions/Finalization"),
61 + metrix.WithUnit("responses"),
62 + ),
63 + duplicateUIDIgnored: meter.Counter(
64 + "duplicate_uid_ignored_total",
65 + metrix.WithDescription("Total number of duplicate transaction IDs ignored at admission"),
66 + metrix.WithChartFamily("Framework/Functions/Admission"),
67 + metrix.WithUnit("requests"),
68 + ),
69 + }
70 +
71 + metrics.invocationsActive.Set(0)
72 + metrics.invocationsAwaitingResult.Set(0)
73 + metrics.schedulerPending.Set(0)
74 +
75 + return metrics
76 +}
77 +
78 +func (m *Manager) observeInvocationsLocked() {
79 + if m == nil || m.runtimeMetrics == nil {
80 + return
81 + }
82 +
83 + active := len(m.invState)
84 + awaiting := 0
85 + for _, rec := range m.invState {
86 + if rec != nil && rec.state == stateAwaitingResult {
87 + awaiting++
88 + }
89 + }
90 +
91 + m.runtimeMetrics.invocationsActive.Set(float64(active))
92 + m.runtimeMetrics.invocationsAwaitingResult.Set(float64(awaiting))
93 +}
94 +
95 +func (m *Manager) observeSchedulerPending() {
96 + if m == nil || m.runtimeMetrics == nil || m.scheduler == nil {
97 + return
98 + }
99 + m.runtimeMetrics.schedulerPending.Set(float64(m.scheduler.pendingCount()))
100 +}
101 +
102 +func (m *Manager) observeQueueFull() {
103 + if m == nil || m.runtimeMetrics == nil {
104 + return
105 + }
106 + m.runtimeMetrics.queueFullTotal.Add(1)
107 +}
108 +
109 +func (m *Manager) observeCancelFallback() {
110 + if m == nil || m.runtimeMetrics == nil {
111 + return
112 + }
113 + m.runtimeMetrics.cancelFallbackTotal.Add(1)
114 +}
115 +
116 +func (m *Manager) observeLateTerminalDropped() {
117 + if m == nil || m.runtimeMetrics == nil {
118 + return
119 + }
120 + m.runtimeMetrics.lateTerminalDropped.Add(1)
121 +}
122 +
123 +func (m *Manager) observeDuplicateUIDIgnored() {
124 + if m == nil || m.runtimeMetrics == nil {
125 + return
126 + }
127 + m.runtimeMetrics.duplicateUIDIgnored.Add(1)
128 +}
src/go/plugin/framework/functions/runtime_metrics_test.go new
+144
@@ -0,0 +1,144 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package functions
4 +
5 +import (
6 + "context"
7 + "sync"
8 + "testing"
9 + "time"
10 +
11 + "github.com/netdata/netdata/go/plugins/pkg/metrix"
12 + "github.com/netdata/netdata/go/plugins/plugin/framework/runtimecomp"
13 + "github.com/stretchr/testify/assert"
14 + "github.com/stretchr/testify/require"
15 +)
16 +
17 +type runtimeServiceMock struct {
18 + mu sync.Mutex
19 +
20 + registered []runtimecomp.ComponentConfig
21 + unregistered []string
22 +}
23 +
24 +func (m *runtimeServiceMock) RegisterComponent(cfg runtimecomp.ComponentConfig) error {
25 + m.mu.Lock()
26 + defer m.mu.Unlock()
27 + m.registered = append(m.registered, cfg)
28 + return nil
29 +}
30 +
31 +func (m *runtimeServiceMock) UnregisterComponent(name string) {
32 + m.mu.Lock()
33 + defer m.mu.Unlock()
34 + m.unregistered = append(m.unregistered, name)
35 +}
36 +
37 +func (m *runtimeServiceMock) RegisterProducer(string, func() error) error { return nil }
38 +
39 +func (m *runtimeServiceMock) snapshot() ([]runtimecomp.ComponentConfig, []string) {
40 + m.mu.Lock()
41 + defer m.mu.Unlock()
42 +
43 + registered := append([]runtimecomp.ComponentConfig(nil), m.registered...)
44 + unregistered := append([]string(nil), m.unregistered...)
45 + return registered, unregistered
46 +}
47 +
48 +func runtimeMetricValue(t *testing.T, store metrix.RuntimeStore, name string, labels metrix.Labels) float64 {
49 + t.Helper()
50 + require.NotNil(t, store)
51 +
52 + reader := store.Read(metrix.ReadRaw())
53 + v, ok := reader.Value(name, labels)
54 + require.Truef(t, ok, "metric %q not found (labels=%v)", name, labels)
55 + return v
56 +}
57 +
58 +func TestManager_RuntimeMetricsScenarios(t *testing.T) {
59 + tests := map[string]struct {
60 + run func(t *testing.T, mgr *Manager, in *chanInput, out *safeBuffer)
61 + }{
62 + "registers and unregisters runtime component around Run lifecycle": {
63 + run: func(t *testing.T, mgr *Manager, in *chanInput, _ *safeBuffer) {
64 + mockSvc := &runtimeServiceMock{}
65 + mgr.SetRuntimeService(mockSvc)
66 + close(in.ch)
67 +
68 + ctx, cancel := context.WithTimeout(context.Background(), time.Second)
69 + defer cancel()
70 + mgr.Run(ctx, nil)
71 +
72 + registered, unregistered := mockSvc.snapshot()
73 + require.Len(t, registered, 1)
74 + assert.Equal(t, functionsRuntimeComponentName, registered[0].Name)
75 + assert.Equal(t, mgr.runtimeStore, registered[0].Store)
76 + assert.True(t, registered[0].Autogen.Enabled)
77 + assert.Equal(t, "functions", registered[0].Module)
78 + assert.Equal(t, "manager", registered[0].JobName)
79 +
80 + require.Len(t, unregistered, 1)
81 + assert.Equal(t, functionsRuntimeComponentName, unregistered[0])
82 + },
83 + },
84 + "pathology counters and gauges are updated": {
85 + run: func(t *testing.T, mgr *Manager, in *chanInput, _ *safeBuffer) {
86 + mgr.workerCount = 1
87 + mgr.queueSize = 1
88 + mgr.cancelFallbackDelay = 50 * time.Millisecond
89 +
90 + started := make(chan struct{}, 1)
91 + release := make(chan struct{})
92 +
93 + mgr.Register("fn", func(fn Function) {
94 + if fn.UID == "tx1" {
95 + started <- struct{}{}
96 + <-release
97 + mgr.respUID(fn.UID, 200, "late")
98 + return
99 + }
100 + mgr.respUID(fn.UID, 200, "ok")
101 + })
102 +
103 + cancel, done := startFlowManager(t, mgr)
104 + defer cancel()
105 +
106 + in.ch <- functionLine("tx1", "fn")
107 + <-started
108 + in.ch <- functionLine("tx2", "fn")
109 + in.ch <- functionLine("tx3", "fn") // queue-full
110 + in.ch <- functionLine("tx1", "fn") // duplicate-active
111 + in.ch <- "FUNCTION_CANCEL tx1" // fallback->499
112 +
113 + waitForCondition(t, time.Second, func() bool {
114 + reader := mgr.runtimeStore.Read(metrix.ReadRaw())
115 + v, ok := reader.Value(functionsRuntimeMetricPrefix+".cancel_fallback_total", nil)
116 + return ok && v >= 1
117 + }, "cancel fallback metric increments")
118 +
119 + in.ch <- functionLine("tx1", "fn") // duplicate-tombstone
120 + close(release)
121 + close(in.ch)
122 + waitForDone(t, done)
123 +
124 + assert.GreaterOrEqual(t, runtimeMetricValue(t, mgr.runtimeStore, functionsRuntimeMetricPrefix+".queue_full_total", nil), float64(1))
125 + assert.GreaterOrEqual(t, runtimeMetricValue(t, mgr.runtimeStore, functionsRuntimeMetricPrefix+".cancel_fallback_total", nil), float64(1))
126 + assert.GreaterOrEqual(t, runtimeMetricValue(t, mgr.runtimeStore, functionsRuntimeMetricPrefix+".late_terminal_dropped_total", nil), float64(1))
127 + assert.GreaterOrEqual(t, runtimeMetricValue(t, mgr.runtimeStore, functionsRuntimeMetricPrefix+".duplicate_uid_ignored_total", nil), float64(2))
128 +
129 + assert.Equal(t, float64(0), runtimeMetricValue(t, mgr.runtimeStore, functionsRuntimeMetricPrefix+".invocations_active", nil))
130 + assert.Equal(t, float64(0), runtimeMetricValue(t, mgr.runtimeStore, functionsRuntimeMetricPrefix+".invocations_awaiting_result", nil))
131 + assert.Equal(t, float64(0), runtimeMetricValue(t, mgr.runtimeStore, functionsRuntimeMetricPrefix+".scheduler_pending", nil))
132 + },
133 + },
134 + }
135 +
136 + for name, tc := range tests {
137 + t.Run(name, func(t *testing.T) {
138 + mgr, out := newFlowManager()
139 + in := &chanInput{ch: make(chan string, 32)}
140 + mgr.input = in
141 + tc.run(t, mgr, in, out)
142 + })
143 + }
144 +}
src/go/plugin/framework/functions/scheduler.go new
+228
@@ -0,0 +1,228 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package functions
4 +
5 +import (
6 + "errors"
7 + "sync"
8 +)
9 +
10 +var (
11 + errSchedulerQueueFull = errors.New("scheduler queue is full")
12 + errSchedulerStopping = errors.New("scheduler is stopping")
13 + errSchedulerInvalid = errors.New("scheduler invalid request")
14 +)
15 +
16 +type scheduleLane struct {
17 + ownerUID string
18 + queue []*invocationRequest
19 +}
20 +
21 +// keyScheduler serializes execution by schedule key while allowing concurrency
22 +// across different keys.
23 +type keyScheduler struct {
24 + mux sync.Mutex
25 +
26 + cond *sync.Cond
27 +
28 + ready []*invocationRequest
29 + lanes map[string]*scheduleLane
30 +
31 + maxPending int
32 + pending int
33 + accepting bool
34 + stopping bool
35 +}
36 +
37 +func newKeyScheduler(maxPending int) *keyScheduler {
38 + s := &keyScheduler{
39 + lanes: make(map[string]*scheduleLane),
40 + maxPending: maxPending,
41 + accepting: true,
42 + }
43 + s.cond = sync.NewCond(&s.mux)
44 + return s
45 +}
46 +
47 +func (s *keyScheduler) enqueue(req *invocationRequest) error {
48 + if req == nil || req.fn == nil || req.fn.UID == "" || req.scheduleKey == "" {
49 + return errSchedulerInvalid
50 + }
51 +
52 + s.mux.Lock()
53 + defer s.mux.Unlock()
54 +
55 + if s.stopping || !s.accepting {
56 + return errSchedulerStopping
57 + }
58 + if s.maxPending > 0 && s.pending >= s.maxPending {
59 + return errSchedulerQueueFull
60 + }
61 +
62 + lane := s.lanes[req.scheduleKey]
63 + if lane == nil {
64 + lane = &scheduleLane{}
65 + s.lanes[req.scheduleKey] = lane
66 + }
67 +
68 + s.pending++
69 + if lane.ownerUID == "" {
70 + lane.ownerUID = req.fn.UID
71 + s.ready = append(s.ready, req)
72 + s.cond.Signal()
73 + return nil
74 + }
75 +
76 + lane.queue = append(lane.queue, req)
77 + return nil
78 +}
79 +
80 +func (s *keyScheduler) next() (*invocationRequest, bool) {
81 + s.mux.Lock()
82 + defer s.mux.Unlock()
83 +
84 + for len(s.ready) == 0 && !s.stopping {
85 + if s.drainedLocked() {
86 + return nil, false
87 + }
88 + s.cond.Wait()
89 + }
90 +
91 + if len(s.ready) == 0 || s.stopping {
92 + return nil, false
93 + }
94 +
95 + req := s.ready[0]
96 + s.ready = s.ready[1:]
97 + if s.pending > 0 {
98 + s.pending--
99 + }
100 + if s.drainedLocked() {
101 + s.cond.Broadcast()
102 + }
103 + return req, true
104 +}
105 +
106 +func (s *keyScheduler) cancelQueued(scheduleKey, uid string) bool {
107 + if scheduleKey == "" || uid == "" {
108 + return false
109 + }
110 +
111 + s.mux.Lock()
112 + defer s.mux.Unlock()
113 +
114 + lane := s.lanes[scheduleKey]
115 + if lane == nil || len(lane.queue) == 0 {
116 + return false
117 + }
118 +
119 + for i, req := range lane.queue {
120 + if req == nil || req.fn == nil || req.fn.UID != uid {
121 + continue
122 + }
123 +
124 + copy(lane.queue[i:], lane.queue[i+1:])
125 + lane.queue = lane.queue[:len(lane.queue)-1]
126 + if s.pending > 0 {
127 + s.pending--
128 + }
129 +
130 + if lane.ownerUID == "" && len(lane.queue) == 0 {
131 + delete(s.lanes, scheduleKey)
132 + }
133 + if s.drainedLocked() {
134 + s.cond.Broadcast()
135 + }
136 + return true
137 + }
138 + return false
139 +}
140 +
141 +func (s *keyScheduler) complete(scheduleKey, uid string) {
142 + if scheduleKey == "" || uid == "" {
143 + return
144 + }
145 +
146 + s.mux.Lock()
147 + defer s.mux.Unlock()
148 +
149 + lane := s.lanes[scheduleKey]
150 + if lane == nil || lane.ownerUID != uid {
151 + return
152 + }
153 +
154 + if s.removeReadyLocked(uid) && s.pending > 0 {
155 + s.pending--
156 + }
157 +
158 + if s.stopping {
159 + delete(s.lanes, scheduleKey)
160 + if s.drainedLocked() {
161 + s.cond.Broadcast()
162 + }
163 + return
164 + }
165 +
166 + for len(lane.queue) > 0 {
167 + next := lane.queue[0]
168 + lane.queue = lane.queue[1:]
169 + if next == nil || next.fn == nil {
170 + if s.pending > 0 {
171 + s.pending--
172 + }
173 + continue
174 + }
175 +
176 + lane.ownerUID = next.fn.UID
177 + s.ready = append(s.ready, next)
178 + s.cond.Signal()
179 + return
180 + }
181 +
182 + lane.ownerUID = ""
183 + if len(lane.queue) == 0 {
184 + delete(s.lanes, scheduleKey)
185 + }
186 + if s.drainedLocked() {
187 + s.cond.Broadcast()
188 + }
189 +}
190 +
191 +func (s *keyScheduler) stopAccepting() {
192 + s.mux.Lock()
193 + s.accepting = false
194 + if s.drainedLocked() {
195 + s.cond.Broadcast()
196 + }
197 + s.mux.Unlock()
198 +}
199 +
200 +func (s *keyScheduler) stop() {
201 + s.mux.Lock()
202 + s.stopping = true
203 + s.cond.Broadcast()
204 + s.mux.Unlock()
205 +}
206 +
207 +func (s *keyScheduler) drainedLocked() bool {
208 + return !s.accepting && s.pending == 0 && len(s.ready) == 0
209 +}
210 +
211 +func (s *keyScheduler) removeReadyLocked(uid string) bool {
212 + for i, req := range s.ready {
213 + if req == nil || req.fn == nil || req.fn.UID != uid {
214 + continue
215 + }
216 +
217 + copy(s.ready[i:], s.ready[i+1:])
218 + s.ready = s.ready[:len(s.ready)-1]
219 + return true
220 + }
221 + return false
222 +}
223 +
224 +func (s *keyScheduler) pendingCount() int {
225 + s.mux.Lock()
226 + defer s.mux.Unlock()
227 + return s.pending
228 +}
src/go/plugin/framework/functions/scheduler_test.go new
+232
@@ -0,0 +1,232 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package functions
4 +
5 +import (
6 + "testing"
7 + "time"
8 +
9 + "github.com/stretchr/testify/assert"
10 + "github.com/stretchr/testify/require"
11 +)
12 +
13 +func TestKeyScheduler_CompleteSkipsInvalidQueuedEntries(t *testing.T) {
14 + tests := map[string]struct {
15 + setup func(*keyScheduler)
16 + check func(*testing.T, *keyScheduler)
17 + }{
18 + "promotes next valid request after invalid queue heads": {
19 + setup: func(s *keyScheduler) {
20 + s.pending = 3
21 + s.lanes["k"] = &scheduleLane{
22 + ownerUID: "owner",
23 + queue: []*invocationRequest{
24 + nil,
25 + {},
26 + {
27 + fn: &Function{UID: "next"},
28 + scheduleKey: "k",
29 + },
30 + },
31 + }
32 + },
33 + check: func(t *testing.T, s *keyScheduler) {
34 + t.Helper()
35 + lane, ok := s.lanes["k"]
36 + require.True(t, ok)
37 + require.NotNil(t, lane)
38 + assert.Equal(t, "next", lane.ownerUID)
39 + assert.Empty(t, lane.queue)
40 + require.Len(t, s.ready, 1)
41 + assert.Equal(t, "next", s.ready[0].fn.UID)
42 + assert.Equal(t, 1, s.pending)
43 +
44 + req, ok := s.next()
45 + require.True(t, ok)
46 + require.NotNil(t, req)
47 + assert.Equal(t, "next", req.fn.UID)
48 + assert.Equal(t, 0, s.pending)
49 + },
50 + },
51 + "drops invalid-only queued entries without pending leak": {
52 + setup: func(s *keyScheduler) {
53 + s.pending = 2
54 + s.lanes["k"] = &scheduleLane{
55 + ownerUID: "owner",
56 + queue: []*invocationRequest{
57 + nil,
58 + {},
59 + },
60 + }
61 + },
62 + check: func(t *testing.T, s *keyScheduler) {
63 + t.Helper()
64 + _, ok := s.lanes["k"]
65 + assert.False(t, ok)
66 + assert.Empty(t, s.ready)
67 + assert.Equal(t, 0, s.pending)
68 + },
69 + },
70 + }
71 +
72 + for name, tc := range tests {
73 + t.Run(name, func(t *testing.T) {
74 + s := newKeyScheduler(10)
75 + tc.setup(s)
76 + s.complete("k", "owner")
77 + tc.check(t, s)
78 + })
79 + }
80 +}
81 +
82 +func TestKeyScheduler_EnqueueValidation(t *testing.T) {
83 + tests := map[string]struct {
84 + req *invocationRequest
85 + adjust func(*keyScheduler)
86 + want error
87 + }{
88 + "nil request returns invalid error": {
89 + req: nil,
90 + want: errSchedulerInvalid,
91 + },
92 + "nil function returns invalid error": {
93 + req: &invocationRequest{
94 + scheduleKey: "k",
95 + },
96 + want: errSchedulerInvalid,
97 + },
98 + "empty uid returns invalid error": {
99 + req: &invocationRequest{
100 + fn: &Function{UID: ""},
101 + scheduleKey: "k",
102 + },
103 + want: errSchedulerInvalid,
104 + },
105 + "empty schedule key returns invalid error": {
106 + req: &invocationRequest{
107 + fn: &Function{UID: "tx1"},
108 + scheduleKey: "",
109 + },
110 + want: errSchedulerInvalid,
111 + },
112 + "stopping scheduler returns stopping error": {
113 + req: &invocationRequest{
114 + fn: &Function{UID: "tx1"},
115 + scheduleKey: "k",
116 + },
117 + adjust: func(s *keyScheduler) { s.stopAccepting() },
118 + want: errSchedulerStopping,
119 + },
120 + "full scheduler returns queue full error": {
121 + req: &invocationRequest{
122 + fn: &Function{UID: "tx1"},
123 + scheduleKey: "k",
124 + },
125 + adjust: func(s *keyScheduler) {
126 + s.pending = 1
127 + },
128 + want: errSchedulerQueueFull,
129 + },
130 + "valid request is admitted": {
131 + req: &invocationRequest{
132 + fn: &Function{UID: "tx1"},
133 + scheduleKey: "k",
134 + },
135 + want: nil,
136 + },
137 + }
138 +
139 + for name, tc := range tests {
140 + t.Run(name, func(t *testing.T) {
141 + s := newKeyScheduler(1)
142 + if tc.adjust != nil {
143 + tc.adjust(s)
144 + }
145 +
146 + err := s.enqueue(tc.req)
147 + if tc.want == nil {
148 + require.NoError(t, err)
149 + return
150 + }
151 + require.ErrorIs(t, err, tc.want)
152 + })
153 + }
154 +}
155 +
156 +func TestKeyScheduler_StopPaths(t *testing.T) {
157 + tests := map[string]struct {
158 + run func(t *testing.T)
159 + }{
160 + "stopAccepting on drained scheduler makes next return false": {
161 + run: func(t *testing.T) {
162 + s := newKeyScheduler(1)
163 + s.stopAccepting()
164 +
165 + req, ok := s.next()
166 + assert.False(t, ok)
167 + assert.Nil(t, req)
168 + },
169 + },
170 + "stop wakes blocked next waiter": {
171 + run: func(t *testing.T) {
172 + s := newKeyScheduler(1)
173 + done := make(chan struct{})
174 +
175 + go func() {
176 + defer close(done)
177 + req, ok := s.next()
178 + assert.False(t, ok)
179 + assert.Nil(t, req)
180 + }()
181 +
182 + time.Sleep(20 * time.Millisecond)
183 + s.stop()
184 +
185 + select {
186 + case <-done:
187 + case <-time.After(time.Second):
188 + t.Fatal("timed out waiting for next() waiter to exit after stop()")
189 + }
190 + },
191 + },
192 + }
193 +
194 + for name, tc := range tests {
195 + t.Run(name, tc.run)
196 + }
197 +}
198 +
199 +func TestKeyScheduler_QueueFullRecovery(t *testing.T) {
200 + tests := map[string]struct {
201 + run func(t *testing.T)
202 + }{
203 + "full queue recovers after dequeue and completion": {
204 + run: func(t *testing.T) {
205 + s := newKeyScheduler(1)
206 + req1 := &invocationRequest{
207 + fn: &Function{UID: "tx1"},
208 + scheduleKey: "k",
209 + }
210 + req2 := &invocationRequest{
211 + fn: &Function{UID: "tx2"},
212 + scheduleKey: "k",
213 + }
214 +
215 + require.NoError(t, s.enqueue(req1))
216 + require.ErrorIs(t, s.enqueue(req2), errSchedulerQueueFull)
217 +
218 + got, ok := s.next()
219 + require.True(t, ok)
220 + require.NotNil(t, got)
221 + require.Equal(t, "tx1", got.fn.UID)
222 +
223 + s.complete("k", "tx1")
224 + require.NoError(t, s.enqueue(req2))
225 + },
226 + },
227 + }
228 +
229 + for name, tc := range tests {
230 + t.Run(name, tc.run)
231 + }
232 +}