crowdsecurity
diff --git a/‎cmd/root.go‎
Lines changed: 2 additions & 8 deletions b/‎cmd/root.go‎
Lines changed: 2 additions & 8 deletions
diff --git a/‎go.mod‎
Lines changed: 1 addition & 0 deletions b/‎go.mod‎
Lines changed: 1 addition & 0 deletions
diff --git a/‎go.sum‎
Lines changed: 2 additions & 0 deletions b/‎go.sum‎
Lines changed: 2 additions & 0 deletions
diff --git a/‎pkg/dataset/bart_types.go‎
Lines changed: 279 additions & 0 deletions b/‎pkg/dataset/bart_types.go‎
Lines changed: 279 additions & 0 deletions
@@ -147,14 +147,8 @@ func Execute() error {
 				if decisions == nil {
 					continue
 				}
-				if len(decisions.New) > 0 {
-					log.Debugf("Processing %d new decisions", len(decisions.New))
-					dataSet.Add(decisions.New)
-				}
-				if len(decisions.Deleted) > 0 {
-					log.Debugf("Processing %d deleted decisions", len(decisions.Deleted))
-					dataSet.Remove(decisions.Deleted)
-				}
+				dataSet.Add(decisions.New)
+				dataSet.Remove(decisions.Deleted)
 			}
 		}
 	})
 
@@ -6,6 +6,7 @@ require (
 	github.com/crowdsecurity/crowdsec v1.7.3
 	github.com/crowdsecurity/go-cs-bouncer v0.0.19
 	github.com/crowdsecurity/go-cs-lib v0.0.23
+	github.com/gaissmai/bart v0.25.0
 	github.com/google/uuid v1.6.0
 	github.com/negasus/haproxy-spoe-go v1.0.7
 	github.com/oschwald/geoip2-golang/v2 v2.0.0
 
@@ -24,6 +24,8 @@ github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0o
 github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/expr-lang/expr v1.17.5 h1:i1WrMvcdLF249nSNlpQZN1S6NXuW9WaOfF5tPi3aw3k=
 github.com/expr-lang/expr v1.17.5/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
+github.com/gaissmai/bart v0.25.0 h1:eqiokVPqM3F94vJ0bTHXHtH91S8zkKL+bKh+BsGOsJM=
+github.com/gaissmai/bart v0.25.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU=
 
@@ -0,0 +1,279 @@
+package dataset
+
+import (
+	"net/netip"
+	"sync"
+	"sync/atomic"
+
+	"github.com/crowdsecurity/crowdsec-spoa/internal/remediation"
+	"github.com/gaissmai/bart"
+	log "github.com/sirupsen/logrus"
+)
+
+// BartAddOp represents a single prefix add operation for batch processing
+type BartAddOp struct {
+	Prefix netip.Prefix
+	Origin string
+	R      remediation.Remediation
+	ID     int64
+	IPType string
+	Scope  string
+}
+
+// BartRemoveOp represents a single prefix removal operation for batch processing
+type BartRemoveOp struct {
+	Prefix netip.Prefix
+	R      remediation.Remediation
+	ID     int64
+	Origin string
+	IPType string
+	Scope  string
+}
+
+// BartUnifiedIPSet provides a unified interface for IP and CIDR operations using bart library.
+// Uses atomic pointer for lock-free reads and mutex-protected writes
+// following the pattern recommended in bart's documentation.
+type BartUnifiedIPSet struct {
+	tableAtomicPtr atomic.Pointer[bart.Table[RemediationIdsMap]]
+	writeMutex     sync.Mutex // Protects writers only
+	logger         *log.Entry
+}
+
+// NewBartUnifiedIPSet creates a new BartUnifiedIPSet
+// The table starts as nil and will be created on first use for better memory efficiency
+// Initialize with nil; the table will be allocated during the first AddBatch operation.
+// This approach enables using Insert for the initial table population, which is more memory efficient than incremental updates.
+func NewBartUnifiedIPSet(logAlias string) *BartUnifiedIPSet {
+	return &BartUnifiedIPSet{
+		logger: log.WithField("alias", logAlias),
+	}
+}
+
+// AddBatch adds multiple prefixes to the bart table in a single atomic operation.
+// This is much more efficient than calling Add() multiple times, as it only
+// swaps the atomic pointer once at the end instead of once per prefix.
+// For the initial load (when table is nil), uses Insert for better memory efficiency.
+// For subsequent updates, uses ModifyPersist for incremental changes.
+// All operations always succeed (duplicates are merged, new entries are created).
+func (s *BartUnifiedIPSet) AddBatch(operations []BartAddOp) {
+	if len(operations) == 0 {
+		return
+	}
+
+	s.writeMutex.Lock()
+	defer s.writeMutex.Unlock()
+
+	// Get current table atomically
+	cur := s.tableAtomicPtr.Load()
+
+	// Check if this is the initial load (table is nil)
+	if cur == nil {
+		s.initializeBatch(operations)
+		return
+	}
+
+	// Table already exists - use ModifyPersist for incremental updates
+	s.updateBatch(cur, operations)
+}
+
+// initializeBatch creates a new table and initializes it with the given operations using Insert.
+// This is more memory efficient than using ModifyPersist for the initial load.
+// Handles duplicate prefixes by merging IDs before inserting.
+// All operations always succeed.
+func (s *BartUnifiedIPSet) initializeBatch(operations []BartAddOp) {
+	// Create a new table for the initial load
+	next := &bart.Table[RemediationIdsMap]{}
+
+	// First, collect all operations by prefix to handle duplicates
+	prefixMap := make(map[netip.Prefix]RemediationIdsMap)
+	for _, op := range operations {
+		prefix := op.Prefix.Masked()
+
+		// Only build logging fields if trace level is enabled
+		var valueLog *log.Entry
+		if s.logger.Logger.IsLevelEnabled(log.TraceLevel) {
+			valueLog = s.logger.WithField("prefix", prefix.String()).WithField("remediation", op.R.String())
+			valueLog.Trace("initial load: collecting prefix operations")
+		}
+
+		// Get or create the data for this prefix
+		data, exists := prefixMap[prefix]
+		if !exists {
+			data = RemediationIdsMap{}
+		}
+		// Add the ID (this handles merging if prefix already seen)
+		data.AddID(valueLog, op.R, op.ID, op.Origin)
+		prefixMap[prefix] = data
+	}
+
+	// Now insert all unique prefixes using Insert
+	for prefix, data := range prefixMap {
+		// Only build logging fields if trace level is enabled
+		var valueLog *log.Entry
+		if s.logger.Logger.IsLevelEnabled(log.TraceLevel) {
+			valueLog = s.logger.WithField("prefix", prefix.String())
+			valueLog.Trace("initial load: inserting into bart trie")
+		}
+
+		// Use Insert for initial load - more memory efficient than ModifyPersist
+		next.Insert(prefix, data)
+	}
+
+	// Atomically swap in the new table
+	s.tableAtomicPtr.Store(next)
+}
+
+// updateBatch updates an existing table with the given operations using ModifyPersist.
+// This handles incremental updates efficiently.
+// All operations always succeed.
+func (s *BartUnifiedIPSet) updateBatch(cur *bart.Table[RemediationIdsMap], operations []BartAddOp) {
+	// Process all operations, chaining the table updates
+	next := cur
+	for _, op := range operations {
+		prefix := op.Prefix.Masked()
+
+		// Only build logging fields if trace level is enabled
+		var valueLog *log.Entry
+		if s.logger.Logger.IsLevelEnabled(log.TraceLevel) {
+			valueLog = s.logger.WithField("prefix", prefix.String()).WithField("remediation", op.R.String())
+			valueLog.Trace("adding to bart trie")
+		}
+
+		// Use ModifyPersist to atomically update or create the prefix entry
+		// This is more efficient than DeletePersist + InsertPersist as it only traverses once
+		next, _, _ = next.ModifyPersist(prefix, func(existingData RemediationIdsMap, exists bool) (RemediationIdsMap, bool) {
+			if exists {
+				if valueLog != nil {
+					valueLog.Trace("exact prefix exists, merging IDs")
+				}
+				// Clone existing data and add the new ID
+				newData := existingData.Clone()
+				newData.AddID(valueLog, op.R, op.ID, op.Origin)
+				return newData, false // false = don't delete
+			}
+			if valueLog != nil {
+				valueLog.Trace("creating new entry")
+			}
+			// Create new data
+			newData := RemediationIdsMap{}
+			newData.AddID(valueLog, op.R, op.ID, op.Origin)
+			return newData, false // false = don't delete
+		})
+	}
+
+	// Atomically swap in the final table (only once for the entire batch)
+	s.tableAtomicPtr.Store(next)
+}
+
+// RemoveBatch removes multiple prefixes from the bart table in a single atomic operation.
+// Returns a slice of pointers to successfully removed operations (nil for failures).
+// This allows callers to access operation metadata (Origin, IPType, Scope) for metrics.
+// IPs should be converted to /32 or /128 prefixes before calling this method.
+func (s *BartUnifiedIPSet) RemoveBatch(operations []BartRemoveOp) []*BartRemoveOp {
+	if len(operations) == 0 {
+		return nil
+	}
+
+	s.writeMutex.Lock()
+	defer s.writeMutex.Unlock()
+
+	// Get current table atomically
+	cur := s.tableAtomicPtr.Load()
+
+	// If table is nil, nothing to remove - return all nil results
+	if cur == nil {
+		return make([]*BartRemoveOp, len(operations))
+	}
+
+	// Process all operations, chaining the table updates
+	next := cur
+	results := make([]*BartRemoveOp, len(operations))
+	for i, op := range operations {
+		prefix := op.Prefix.Masked()
+
+		// Only build logging fields if trace level is enabled
+		var valueLog *log.Entry
+		if s.logger.Logger.IsLevelEnabled(log.TraceLevel) {
+			valueLog = s.logger.WithField("prefix", prefix.String()).WithField("remediation", op.R.String())
+			valueLog.Trace("removing from bart trie")
+		}
+
+		// Use ModifyPersist to atomically update or remove the prefix entry
+		// This is more efficient than DeletePersist + InsertPersist as it only traverses once
+		next, _, _ = next.ModifyPersist(prefix, func(existingData RemediationIdsMap, exists bool) (RemediationIdsMap, bool) {
+			if !exists {
+				if valueLog != nil {
+					valueLog.Trace("exact prefix not found")
+				}
+				results[i] = nil
+				return existingData, false // false = don't delete (prefix doesn't exist anyway)
+			}
+
+			// Clone existing data and remove the ID
+			clonedData := existingData.Clone()
+			err := clonedData.RemoveID(valueLog, op.R, op.ID)
+			if err != nil {
+				if valueLog != nil {
+					valueLog.Trace("ID not found")
+				}
+				results[i] = nil
+				return existingData, false // false = don't delete, keep original data
+			}
+
+			// ID was successfully removed - return pointer to the operation for metadata access
+			// Use index to get pointer to original operation (safe since we don't modify the slice)
+			results[i] = &operations[i]
+
+			if clonedData.IsEmpty() {
+				if valueLog != nil {
+					valueLog.Trace("removed prefix entirely")
+				}
+				return clonedData, true // true = delete the prefix (it's now empty)
+			}
+			if valueLog != nil {
+				valueLog.Trace("removed ID from existing prefix")
+			}
+			return clonedData, false // false = don't delete, keep modified data
+		})
+	}
+
+	// Atomically swap in the final table (only once for the entire batch)
+	s.tableAtomicPtr.Store(next)
+
+	return results
+}
+
+// Contains checks if an IP address matches any prefix in the bart table.
+// Returns the longest matching prefix's remediation and origin.
+// This method uses lock-free reads via atomic pointer for optimal performance.
+func (s *BartUnifiedIPSet) Contains(ip netip.Addr) (remediation.Remediation, string) {
+	// Lock-free read: atomically load the current table pointer
+	table := s.tableAtomicPtr.Load()
+
+	// Check for nil table (not yet initialized)
+	if table == nil {
+		return remediation.Allow, ""
+	}
+
+	// Only build logging fields if trace level is enabled
+	var valueLog *log.Entry
+	if s.logger.Logger.IsLevelEnabled(log.TraceLevel) {
+		valueLog = s.logger.WithField("ip", ip.String())
+		valueLog.Trace("checking in bart trie")
+	}
+
+	// Use Lookup to get the longest prefix match
+	data, found := table.Lookup(ip)
+	if !found {
+		if valueLog != nil {
+			valueLog.Trace("no match found")
+		}
+		return remediation.Allow, ""
+	}
+
+	remediationResult, origin := data.GetRemediationAndOrigin()
+	if valueLog != nil {
+		valueLog.Tracef("bart result: %s (data: %+v)", remediationResult.String(), data)
+	}
+	return remediationResult, origin
+}
Original file line number	Diff line number	Diff line change
`@@ -147,14 +147,8 @@ func Execute() error {`
`147`	`147`	`if decisions == nil {`
`148`	`148`	`continue`
`149`	`149`	`}`
`150`		`- if len(decisions.New) > 0 {`
`151`		`- log.Debugf("Processing %d new decisions", len(decisions.New))`
`152`		`- dataSet.Add(decisions.New)`
`153`		`- }`
`154`		`- if len(decisions.Deleted) > 0 {`
`155`		`- log.Debugf("Processing %d deleted decisions", len(decisions.Deleted))`
`156`		`- dataSet.Remove(decisions.Deleted)`
`157`		`- }`
	`150`	`+ dataSet.Add(decisions.New)`
	`151`	`+ dataSet.Remove(decisions.Deleted)`
`158`	`152`	`}`
`159`	`153`	`}`
`160`	`154`	`})`