implement Rule-Aware Profile Projection

.github/workflows/component-tests.yaml

@@ -101,9 +101,9 @@ jobs:
         run: |
           STORAGE_TAG=$(./tests/scripts/storage-tag.sh)
           echo "Storage tag that will be used: ${STORAGE_TAG}"
-          helm upgrade --install kubescape ./tests/chart --set clusterName=`kubectl config current-context` --set nodeAgent.image.tag=${{ needs.build-and-push-image.outputs.image_tag }} --set nodeAgent.image.repository=${{ needs.build-and-push-image.outputs.image_repo }} --set storage.image.tag=${STORAGE_TAG} -n kubescape --create-namespace --wait --timeout 5m --debug
+          helm upgrade --install kubescape ./tests/chart --set clusterName=`kubectl config current-context` --set nodeAgent.image.tag=${{ needs.build-and-push-image.outputs.image_tag }} --set nodeAgent.image.repository=${{ needs.build-and-push-image.outputs.image_repo }} --set storage.image.tag=${STORAGE_TAG} -n kubescape --create-namespace --wait --timeout 10m --debug
           # Check that the node-agent pod is running
-          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=node-agent -n kubescape --timeout=300s
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=node-agent -n kubescape --timeout=600s
           sleep 5
       - name: Run Port Forwarding
         run: |

README.md

@@ -344,6 +344,12 @@ spec:
 - **Crypto Rules**: Mining activity detection via RandomX
 - **Container Rules**: Escape attempts, namespace manipulation
 
+### CEL Helper Limitations (v1)
+
+| Helper | v1 Behaviour | Note |
+|--------|-------------|------|
+| `wasExecutedWithArgs(containerID, path, args)` | Equivalent to `wasExecuted(containerID, path)` — the `args` list is validated for type correctness but is **not** matched against the recorded argument list. Any execution of the given path returns `true` regardless of its arguments. | Full per-argument matching (`ExecArgsByPath`) will be added in a future version. |
+
 For the full list of rules, see the [Kubescape documentation](https://kubescape.io/docs/).
 
 ## 🎮 Demos & Examples

cmd/main.go

@@ -309,7 +309,7 @@ func main() {
 
 		adapterFactory := ruleadapters.NewEventRuleAdapterFactory()
 
-		celEvaluator, err := cel.NewCEL(objCache, cfg)
+		celEvaluator, err := cel.NewCEL(objCache, cfg, prometheusExporter)
 		if err != nil {
 			logger.L().Ctx(ctx).Fatal("error creating CEL evaluator", helpers.Error(err))
 		}

go.mod

@@ -55,10 +55,12 @@ require (
 	go.uber.org/multierr v1.11.0
 	golang.org/x/net v0.53.0
 	golang.org/x/sys v0.43.0
+	golang.org/x/tools v0.43.0
 	gonum.org/v1/plot v0.14.0
 	google.golang.org/grpc v1.80.0
 	google.golang.org/protobuf v1.36.11
 	gopkg.in/mcuadros/go-syslog.v2 v2.3.0
+	gopkg.in/yaml.v3 v3.0.1
 	istio.io/pkg v0.0.0-20231221211216-7635388a563e
 	k8s.io/api v0.35.0
 	k8s.io/apimachinery v0.35.0
@@ -435,7 +437,6 @@ require (
 	golang.org/x/term v0.42.0 // indirect
 	golang.org/x/text v0.36.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
-	golang.org/x/tools v0.43.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.271.0 // indirect
 	google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
@@ -445,7 +446,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.35.0 // indirect
 	k8s.io/apiserver v0.35.0 // indirect
 	k8s.io/cli-runtime v0.35.0 // indirect

pkg/config/config.go

@@ -30,6 +30,15 @@ const PodNameEnvVar = "POD_NAME"
 const NamespaceEnvVar = "NAMESPACE_NAME"
 
 // EventDedupConfig controls eBPF event deduplication before CEL rule evaluation.
+// ProfileProjectionConfig controls rule-aware profile projection behaviour.
+type ProfileProjectionConfig struct {
+	// DetailedMetricsEnabled enables per-rule stale-entry and literal-miss counters.
+	DetailedMetricsEnabled bool `mapstructure:"detailedMetricsEnabled"`
+	// StrictValidation rejects rules with profileDependency>0 but no profileDataRequired.
+	// Defaults to false (soft mode: log + metric only).
+	StrictValidation bool `mapstructure:"strictValidation"`
+}
+
 type EventDedupConfig struct {
 	Enabled       bool  `mapstructure:"enabled"`
 	SlotsExponent uint8 `mapstructure:"slotsExponent"`
@@ -105,6 +114,7 @@ type Config struct {
 	PodName                        string                               `mapstructure:"podName"`
 	ProcfsPidScanInterval          time.Duration                        `mapstructure:"procfsPidScanInterval"`
 	ProcfsScanInterval             time.Duration                        `mapstructure:"procfsScanInterval"`
+	ProfileProjection              ProfileProjectionConfig              `mapstructure:"profileProjection"`
 	ProfilesCacheRefreshRate       time.Duration                        `mapstructure:"profilesCacheRefreshRate"`
 	StorageRPCBudget               time.Duration                        `mapstructure:"storageRPCBudget"`
 	RuleCoolDown                   rulecooldown.RuleCooldownConfig      `mapstructure:"ruleCooldown"`

pkg/metricsmanager/metrics_manager_interface.go

@@ -25,4 +25,27 @@ type MetricsManager interface {
 	ReportContainerProfileCacheHit(hit bool)
 	ReportContainerProfileReconcilerDuration(phase string, duration time.Duration)
 	ReportContainerProfileReconcilerEviction(reason string)
+
+	// Profile-projection metrics — always-on.
+	IncMissingProfileDataRequired(ruleID string)      // rule has profileDependency>0 but no profileDataRequired
+	IncProjectionUndeclaredLiteral(helper string)     // literal evaluated against a projected field not in spec
+	SetProjectionStaleEntries(count float64)          // cache entries whose SpecHash != currentSpecHash
+	SetProjectionUndeclaredRules(count float64)       // rules loaded with no profileDataRequired
+
+	// Profile-projection metrics — detailed (gated by profileProjection.detailedMetricsEnabled).
+	IncProjectionSpecCompile()
+	IncProjectionSpecHashChange()
+	SetProjectionSpecPatterns(field, kind string, count float64)
+	SetProjectionSpecAllField(field string, isAll bool)
+	ObserveProjectionApplyDuration(d time.Duration)
+	IncProjectionReconcileTriggered(trigger string)
+	IncHelperCall(helper string)
+	SetProjectionUndeclaredRulesDetail(ruleIDs []string)
+
+	// Memory-savings metrics — detailed (gated by profileProjection.detailedMetricsEnabled).
+	ObserveProfileRawSize(bytes float64)
+	ObserveProfileProjectedSize(bytes float64)
+	ObserveProfileEntriesRaw(field string, count float64)
+	ObserveProfileEntriesRetained(field string, count float64)
+	ObserveProfileRetentionRatio(field string, ratio float64)
 }

pkg/metricsmanager/metrics_manager_mock.go

@@ -72,3 +72,20 @@ func (m *MetricsMock) SetContainerProfileCacheEntries(_ string, _ float64)
 func (m *MetricsMock) ReportContainerProfileCacheHit(_ bool)                              {}
 func (m *MetricsMock) ReportContainerProfileReconcilerDuration(_ string, _ time.Duration) {}
 func (m *MetricsMock) ReportContainerProfileReconcilerEviction(_ string)                  {}
+func (m *MetricsMock) IncMissingProfileDataRequired(_ string)            {}
+func (m *MetricsMock) IncProjectionUndeclaredLiteral(_ string)           {}
+func (m *MetricsMock) SetProjectionStaleEntries(_ float64)               {}
+func (m *MetricsMock) SetProjectionUndeclaredRules(_ float64)            {}
+func (m *MetricsMock) IncProjectionSpecCompile()                         {}
+func (m *MetricsMock) IncProjectionSpecHashChange()                      {}
+func (m *MetricsMock) SetProjectionSpecPatterns(_, _ string, _ float64)  {}
+func (m *MetricsMock) SetProjectionSpecAllField(_ string, _ bool)        {}
+func (m *MetricsMock) ObserveProjectionApplyDuration(_ time.Duration)    {}
+func (m *MetricsMock) IncProjectionReconcileTriggered(_ string)          {}
+func (m *MetricsMock) IncHelperCall(_ string)                            {}
+func (m *MetricsMock) SetProjectionUndeclaredRulesDetail(_ []string)     {}
+func (m *MetricsMock) ObserveProfileRawSize(_ float64)                   {}
+func (m *MetricsMock) ObserveProfileProjectedSize(_ float64)             {}
+func (m *MetricsMock) ObserveProfileEntriesRaw(_ string, _ float64)     {}
+func (m *MetricsMock) ObserveProfileEntriesRetained(_ string, _ float64) {}
+func (m *MetricsMock) ObserveProfileRetentionRatio(_ string, _ float64)  {}

pkg/metricsmanager/metrics_manager_noop.go

@@ -27,3 +27,20 @@ func (m *MetricsNoop) SetContainerProfileCacheEntries(_ string, _ float64)
 func (m *MetricsNoop) ReportContainerProfileCacheHit(_ bool)                               {}
 func (m *MetricsNoop) ReportContainerProfileReconcilerDuration(_ string, _ time.Duration)  {}
 func (m *MetricsNoop) ReportContainerProfileReconcilerEviction(_ string)                   {}
+func (m *MetricsNoop) IncMissingProfileDataRequired(_ string)            {}
+func (m *MetricsNoop) IncProjectionUndeclaredLiteral(_ string)           {}
+func (m *MetricsNoop) SetProjectionStaleEntries(_ float64)               {}
+func (m *MetricsNoop) SetProjectionUndeclaredRules(_ float64)            {}
+func (m *MetricsNoop) IncProjectionSpecCompile()                         {}
+func (m *MetricsNoop) IncProjectionSpecHashChange()                      {}
+func (m *MetricsNoop) SetProjectionSpecPatterns(_, _ string, _ float64)  {}
+func (m *MetricsNoop) SetProjectionSpecAllField(_ string, _ bool)        {}
+func (m *MetricsNoop) ObserveProjectionApplyDuration(_ time.Duration)    {}
+func (m *MetricsNoop) IncProjectionReconcileTriggered(_ string)          {}
+func (m *MetricsNoop) IncHelperCall(_ string)                            {}
+func (m *MetricsNoop) SetProjectionUndeclaredRulesDetail(_ []string)     {}
+func (m *MetricsNoop) ObserveProfileRawSize(_ float64)                   {}
+func (m *MetricsNoop) ObserveProfileProjectedSize(_ float64)             {}
+func (m *MetricsNoop) ObserveProfileEntriesRaw(_ string, _ float64)     {}
+func (m *MetricsNoop) ObserveProfileEntriesRetained(_ string, _ float64) {}
+func (m *MetricsNoop) ObserveProfileRetentionRatio(_ string, _ float64)  {}

pkg/metricsmanager/prometheus/prometheus.go

@@ -70,6 +70,29 @@ type PrometheusMetric struct {
 	cpReconcilerDurationHistogram  *prometheus.HistogramVec
 	cpReconcilerEvictionsCounter   *prometheus.CounterVec
 
+	// Profile projection metrics — always-on
+	cpProjectionMissingDeclCounter      *prometheus.CounterVec
+	cpProjectionUndeclaredLiteralCounter *prometheus.CounterVec
+	cpProjectionStaleEntriesGauge        prometheus.Gauge
+	cpProjectionUndeclaredRulesGauge     prometheus.Gauge
+
+	// Profile projection metrics — detailed (gated by caller checking detailedMetricsEnabled)
+	cpProjectionSpecCompileCounter        prometheus.Counter
+	cpProjectionSpecHashChangeCounter     prometheus.Counter
+	cpProjectionSpecPatternsGauge         *prometheus.GaugeVec
+	cpProjectionSpecAllFieldsGauge        *prometheus.GaugeVec
+	cpProjectionApplyDurationHistogram    prometheus.Histogram
+	cpProjectionReconcileTriggeredCounter *prometheus.CounterVec
+	cpHelperCallCounter                   *prometheus.CounterVec
+	cpProjectionUndeclaredRulesListGauge  *prometheus.GaugeVec
+
+	// Memory-savings metrics — detailed
+	cpProfileRawSizeHistogram         prometheus.Histogram
+	cpProfileProjectedSizeHistogram   prometheus.Histogram
+	cpProfileEntriesRawHistogram      *prometheus.HistogramVec
+	cpProfileEntriesRetainedHistogram *prometheus.HistogramVec
+	cpProfileRetentionRatioHistogram  *prometheus.HistogramVec
+
 	// Cache to avoid allocating Labels maps on every call
 	ruleCounterCache          map[string]prometheus.Counter
 	rulePrefilteredCounterCache map[string]prometheus.Counter
@@ -245,6 +268,86 @@ func NewPrometheusMetric() *PrometheusMetric {
 			Help: "Total number of ContainerProfile cache evictions by reason.",
 		}, []string{"reason"}),
 
+		// Profile projection metrics — always-on
+		cpProjectionMissingDeclCounter: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "rule_load_rejected_missing_declaration_total",
+			Help: "Total rules with profileDependency>0 but no profileDataRequired declaration.",
+		}, []string{"rule_id"}),
+		cpProjectionUndeclaredLiteralCounter: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "rule_projection_undeclared_literal_total",
+			Help: "Total literal values evaluated against a projected field that was not declared.",
+		}, []string{"helper"}),
+		cpProjectionStaleEntriesGauge: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "rule_projection_stale_entries",
+			Help: "Current number of projected cache entries whose spec hash is stale.",
+		}),
+		cpProjectionUndeclaredRulesGauge: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "rule_projection_undeclared_rules",
+			Help: "Currently-loaded rules with no profileDataRequired field.",
+		}),
+
+		// Profile projection metrics — detailed
+		cpProjectionSpecCompileCounter: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "rule_projection_spec_compile_total",
+			Help: "Total number of times the projection spec was compiled.",
+		}),
+		cpProjectionSpecHashChangeCounter: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "rule_projection_spec_hash_changes_total",
+			Help: "Total number of times the projection spec hash changed.",
+		}),
+		cpProjectionSpecPatternsGauge: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "rule_projection_spec_patterns",
+			Help: "Number of patterns per field and kind in the current projection spec.",
+		}, []string{"field", "kind"}),
+		cpProjectionSpecAllFieldsGauge: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "rule_projection_spec_all_fields",
+			Help: "Whether a projection spec field has All=true (1) or not (0).",
+		}, []string{"field"}),
+		cpProjectionApplyDurationHistogram: promauto.NewHistogram(prometheus.HistogramOpts{
+			Name:    "rule_projection_apply_duration_seconds",
+			Help:    "Duration of profile projection Apply calls in seconds.",
+			Buckets: prometheus.DefBuckets,
+		}),
+		cpProjectionReconcileTriggeredCounter: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "rule_projection_reconcile_triggered_total",
+			Help: "Total number of projection reconcile triggers by type.",
+		}, []string{"trigger"}),
+		cpHelperCallCounter: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "rule_helper_call_total",
+			Help: "Total number of profile-helper CEL function calls.",
+		}, []string{"helper"}),
+		cpProjectionUndeclaredRulesListGauge: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "rule_projection_undeclared_rules_list",
+			Help: "Per-rule gauge (1) for each rule currently loaded without a profileDataRequired declaration.",
+		}, []string{"rule_id"}),
+
+		// Memory-savings metrics — detailed
+		cpProfileRawSizeHistogram: promauto.NewHistogram(prometheus.HistogramOpts{
+			Name:    "profile_raw_size_bytes",
+			Help:    "Approximate byte size of raw ContainerProfile string data before projection.",
+			Buckets: []float64{0, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304},
+		}),
+		cpProfileProjectedSizeHistogram: promauto.NewHistogram(prometheus.HistogramOpts{
+			Name:    "profile_projected_size_bytes",
+			Help:    "Approximate byte size of projected ContainerProfile string data after projection.",
+			Buckets: []float64{0, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304},
+		}),
+		cpProfileEntriesRawHistogram: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "profile_entries_raw_total",
+			Help:    "Number of entries per field in the raw profile before projection.",
+			Buckets: []float64{0, 1, 5, 10, 50, 100, 500, 1000, 5000},
+		}, []string{"field"}),
+		cpProfileEntriesRetainedHistogram: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "profile_entries_retained_total",
+			Help:    "Number of entries per field retained after projection.",
+			Buckets: []float64{0, 1, 5, 10, 50, 100, 500, 1000, 5000},
+		}, []string{"field"}),
+		cpProfileRetentionRatioHistogram: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "profile_retention_ratio",
+			Help:    "Fraction of entries retained per field after projection (retained/raw).",
+			Buckets: []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0},
+		}, []string{"field"}),
+
 		// Initialize counter caches
 		ruleCounterCache:            make(map[string]prometheus.Counter),
 		rulePrefilteredCounterCache: make(map[string]prometheus.Counter),
@@ -291,6 +394,23 @@ func (p *PrometheusMetric) Destroy() {
 	prometheus.Unregister(p.cpCacheHitCounter)
 	prometheus.Unregister(p.cpReconcilerDurationHistogram)
 	prometheus.Unregister(p.cpReconcilerEvictionsCounter)
+	prometheus.Unregister(p.cpProjectionMissingDeclCounter)
+	prometheus.Unregister(p.cpProjectionUndeclaredLiteralCounter)
+	prometheus.Unregister(p.cpProjectionStaleEntriesGauge)
+	prometheus.Unregister(p.cpProjectionUndeclaredRulesGauge)
+	prometheus.Unregister(p.cpProjectionSpecCompileCounter)
+	prometheus.Unregister(p.cpProjectionSpecHashChangeCounter)
+	prometheus.Unregister(p.cpProjectionSpecPatternsGauge)
+	prometheus.Unregister(p.cpProjectionSpecAllFieldsGauge)
+	prometheus.Unregister(p.cpProjectionApplyDurationHistogram)
+	prometheus.Unregister(p.cpProjectionReconcileTriggeredCounter)
+	prometheus.Unregister(p.cpHelperCallCounter)
+	prometheus.Unregister(p.cpProjectionUndeclaredRulesListGauge)
+	prometheus.Unregister(p.cpProfileRawSizeHistogram)
+	prometheus.Unregister(p.cpProfileProjectedSizeHistogram)
+	prometheus.Unregister(p.cpProfileEntriesRawHistogram)
+	prometheus.Unregister(p.cpProfileEntriesRetainedHistogram)
+	prometheus.Unregister(p.cpProfileRetentionRatioHistogram)
 	// Unregister program ID metrics
 	prometheus.Unregister(p.programRuntimeGauge)
 	prometheus.Unregister(p.programRunCountGauge)
@@ -491,3 +611,62 @@ func (p *PrometheusMetric) ReportContainerProfileReconcilerDuration(phase string
 func (p *PrometheusMetric) ReportContainerProfileReconcilerEviction(reason string) {
 	p.cpReconcilerEvictionsCounter.WithLabelValues(reason).Inc()
 }
+
+func (p *PrometheusMetric) IncMissingProfileDataRequired(ruleID string) {
+	p.cpProjectionMissingDeclCounter.WithLabelValues(ruleID).Inc()
+}
+func (p *PrometheusMetric) IncProjectionUndeclaredLiteral(helper string) {
+	p.cpProjectionUndeclaredLiteralCounter.WithLabelValues(helper).Inc()
+}
+func (p *PrometheusMetric) SetProjectionStaleEntries(count float64) {
+	p.cpProjectionStaleEntriesGauge.Set(count)
+}
+func (p *PrometheusMetric) SetProjectionUndeclaredRules(count float64) {
+	p.cpProjectionUndeclaredRulesGauge.Set(count)
+}
+func (p *PrometheusMetric) IncProjectionSpecCompile() {
+	p.cpProjectionSpecCompileCounter.Inc()
+}
+func (p *PrometheusMetric) IncProjectionSpecHashChange() {
+	p.cpProjectionSpecHashChangeCounter.Inc()
+}
+func (p *PrometheusMetric) SetProjectionSpecPatterns(field, kind string, count float64) {
+	p.cpProjectionSpecPatternsGauge.WithLabelValues(field, kind).Set(count)
+}
+func (p *PrometheusMetric) SetProjectionSpecAllField(field string, isAll bool) {
+	v := float64(0)
+	if isAll {
+		v = 1
+	}
+	p.cpProjectionSpecAllFieldsGauge.WithLabelValues(field).Set(v)
+}
+func (p *PrometheusMetric) ObserveProjectionApplyDuration(d time.Duration) {
+	p.cpProjectionApplyDurationHistogram.Observe(d.Seconds())
+}
+func (p *PrometheusMetric) IncProjectionReconcileTriggered(trigger string) {
+	p.cpProjectionReconcileTriggeredCounter.WithLabelValues(trigger).Inc()
+}
+func (p *PrometheusMetric) IncHelperCall(helper string) {
+	p.cpHelperCallCounter.WithLabelValues(helper).Inc()
+}
+func (p *PrometheusMetric) SetProjectionUndeclaredRulesDetail(ruleIDs []string) {
+	p.cpProjectionUndeclaredRulesListGauge.Reset()
+	for _, id := range ruleIDs {
+		p.cpProjectionUndeclaredRulesListGauge.WithLabelValues(id).Set(1)
+	}
+}
+func (p *PrometheusMetric) ObserveProfileRawSize(bytes float64) {
+	p.cpProfileRawSizeHistogram.Observe(bytes)
+}
+func (p *PrometheusMetric) ObserveProfileProjectedSize(bytes float64) {
+	p.cpProfileProjectedSizeHistogram.Observe(bytes)
+}
+func (p *PrometheusMetric) ObserveProfileEntriesRaw(field string, count float64) {
+	p.cpProfileEntriesRawHistogram.WithLabelValues(field).Observe(count)
+}
+func (p *PrometheusMetric) ObserveProfileEntriesRetained(field string, count float64) {
+	p.cpProfileEntriesRetainedHistogram.WithLabelValues(field).Observe(count)
+}
+func (p *PrometheusMetric) ObserveProfileRetentionRatio(field string, ratio float64) {
+	p.cpProfileRetentionRatioHistogram.WithLabelValues(field).Observe(ratio)
+}