diff --git a/cmd/packet_capture.go b/cmd/packet_capture.go index 3b1cca1c9..78c7ffea9 100644 --- a/cmd/packet_capture.go +++ b/cmd/packet_capture.go @@ -1,11 +1,15 @@ package cmd import ( + "bufio" "encoding/base64" "encoding/json" "fmt" + "io" + "os" "sort" "strings" + "sync" "time" "github.com/gopacket/gopacket" @@ -30,12 +34,19 @@ var ( srcComment strings.Builder dstComment strings.Builder commonComment strings.Builder + tlsKeylogPath string ) +func init() { + pktCmd.Flags().StringVar(&tlsKeylogPath, "tls-keylog", "", "Path to TLS key log file (SSLKEYLOGFILE format) for pcapng DSB") +} + func runPacketCapture(_ *cobra.Command, _ []string) { capture = Packet + showCount = defaultFlowShowCount + clearPacketCaptureBuffers() if isBackground { - go backgroundHearbeat() // show table periodically in background + go backgroundHearbeat() startPacketCollector() } else { go startPacketCollector() @@ -43,7 +54,6 @@ func runPacketCapture(_ *cobra.Command, _ []string) { } } -//nolint:cyclop func startPacketCollector() { if len(filename) > 0 { log.Infof("Starting Packet Capture for %s...", filename) @@ -51,7 +61,7 @@ func startPacketCollector() { log.Infof("Starting Packet Capture...") filename = strings.ReplaceAll( currentTime().UTC().Format(time.RFC3339), - ":", "") // get rid of offensive colons + ":", "") } f, err := createOutputFile("pcap", filename+".pcapng") @@ -59,7 +69,17 @@ func startPacketCollector() { log.Fatal(err) } defer f.Close() - log.Trace("Created pcapng file") + + var plaintextLog io.WriteCloser + if plaintextCaptureEnabled() { + plaintextFile, err := createOutputFile("plaintext", filename+".jsonl") + if err != nil { + log.Error("failed to create plaintext log", err) + } else { + plaintextLog = plaintextFile + defer plaintextLog.Close() + } + } ngw, err := pcapgo.NewNgWriter(f, layers.LinkTypeEthernet) if err != nil { @@ -67,7 +87,13 @@ func startPacketCollector() { return } defer ngw.Flush() - log.Trace("Wrote pcap section header & interface") + + if tlsKeylogPath != "" { + if err := embedTLSKeylog(ngw, tlsKeylogPath); err != nil { + log.Warnf("TLS keylog embed failed: %v", err) + } + go watchTLSKeylog(ngw, tlsKeylogPath) + } flowPackets := make(chan *genericmap.Flow, 100) collector, err := grpc.StartCollector(port, flowPackets) @@ -80,49 +106,39 @@ func startPacketCollector() { go func() { <-utils.ExitChannel() - log.Debug("Ending collector") close(flowPackets) collector.Close() - log.Debug("Done") }() - log.Trace("Ready ! Waiting for packets...") for fp := range flowPackets { - if !captureStarted { - log.Debugf("Received first %d packets", len(flowPackets)) - } - if stopReceived { - log.Debug("Stop received") return } genericMap := config.GenericMap{} - err := json.Unmarshal(fp.GenericMap.Value, &genericMap) - if err != nil { + if err := json.Unmarshal(fp.GenericMap.Value, &genericMap); err != nil { log.Error("Error while parsing json", err) return } - if !captureStarted { - log.Debugf("Parsed genericMap %v", genericMap) + + if isPlaintextRecord(genericMap) { + assignPlaintextPacketID(&genericMap) + enrichPlaintextForExport(&genericMap) + genericMap["PcapAnnotated"] = false + if plaintextLog != nil { + writePlaintextJSONL(plaintextLog, &genericMap) + } + continue } data, ok := genericMap["Data"] if ok { - // display as flow async go AppendFlow(genericMap.Copy()) - writePacketData(ngw, &genericMap, &data) } else { - if !captureStarted { - log.Debug("Data is missing") - } - - // display as flow async go AppendFlow(genericMap) } - // terminate capture if max bytes reached totalBytes += int64(len(fp.GenericMap.Value)) if totalBytes > maxBytes { if exit := onLimitReached(); exit { @@ -131,10 +147,8 @@ func startPacketCollector() { } } - // terminate capture if max time reached now := currentTime() - duration := now.Sub(startupTime) - if int(duration) > int(maxTime) { + if int(now.Sub(startupTime)) > int(maxTime) { if exit := onLimitReached(); exit { log.Infof("Capture reached %s, exiting now...", maxTime) return @@ -145,35 +159,51 @@ func startPacketCollector() { } } +func plaintextCaptureEnabled() bool { + return optionEnabled("enable_openssl") +} + +// clearPacketCaptureBuffers is a no-op until wire/TUI correlation lands (NETOBSERV-2859). +func clearPacketCaptureBuffers() {} + +func isPlaintextRecord(m config.GenericMap) bool { + rt, ok := m["RecordType"].(string) + return ok && rt == "plaintext" +} + +func writePlaintextJSONL(w io.Writer, m *config.GenericMap) { + line, err := json.Marshal(m) + if err != nil { + log.Error("plaintext json marshal", err) + return + } + if _, err := w.Write(append(line, '\n')); err != nil { + log.Error("plaintext json write", err) + } +} + func writePacketData(ngw *pcapgo.NgWriter, genericMap *config.GenericMap, data *interface{}) { - // Get capture timestamp ts := time.Unix(int64((*genericMap)["Time"].(float64)), 0) - // Decode b64 encoded data b, err := base64.StdEncoding.DecodeString((*data).(string)) if err != nil { log.Error("Error while decoding data", err) return } - // sort generic map keys to keep comments ordered keys := make([]string, 0, len((*genericMap))) for k := range *genericMap { - // ignore time field if k == "Time" || k == "Data" { continue } keys = append(keys, k) - } sort.Strings(keys) - // generate comments per category srcComment.WriteString("Source\n") dstComment.WriteString("Destination\n") commonComment.WriteString("Common\n") for _, k := range keys { id := toColID(k) - // add name and value without truncating text str := fmt.Sprintf("%s: %v\n", toColName(id, 0), toColValue((*genericMap), id, 0)) if strings.HasPrefix(k, "Src") { srcComment.WriteString(str) @@ -184,7 +214,6 @@ func writePacketData(ngw *pcapgo.NgWriter, genericMap *config.GenericMap, data * } } - // write enriched data as interface if err := ngw.WritePacketWithOptions(gopacket.CaptureInfo{ Timestamp: ts, Length: len(b), @@ -204,3 +233,68 @@ func writePacketData(ngw *pcapgo.NgWriter, genericMap *config.GenericMap, data * dstComment.Reset() commonComment.Reset() } + +var keylogMu sync.Mutex +var keylogOffset int64 + +func embedTLSKeylog(ngw *pcapgo.NgWriter, path string) error { + content, err := os.ReadFile(path) + if err != nil { + return err + } + if len(content) == 0 { + return nil + } + keylogMu.Lock() + defer keylogMu.Unlock() + if err := ngw.WriteDecryptionSecretsBlock(pcapgo.DSB_SECRETS_TYPE_TLS, content); err != nil { + return err + } + keylogOffset = int64(len(content)) + return nil +} + +func watchTLSKeylog(ngw *pcapgo.NgWriter, path string) { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for range ticker.C { + if stopReceived { + return + } + f, err := os.Open(path) + if err != nil { + continue + } + if _, err := f.Seek(keylogOffset, io.SeekStart); err != nil { + _ = f.Close() + continue + } + data, err := io.ReadAll(f) + _ = f.Close() + if err != nil || len(data) == 0 { + continue + } + keylogMu.Lock() + if err := ngw.WriteDecryptionSecretsBlock(pcapgo.DSB_SECRETS_TYPE_TLS, data); err != nil { + log.Warnf("failed to append TLS keylog DSB: %v", err) + } else { + keylogOffset += int64(len(data)) + } + keylogMu.Unlock() + } +} + +// ParseKeylogLines reads NSS key log format lines from a reader. +func ParseKeylogLines(r io.Reader) ([]byte, error) { + var buf strings.Builder + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + buf.WriteString(line) + buf.WriteByte('\n') + } + return []byte(buf.String()), scanner.Err() +} diff --git a/cmd/packet_capture_plaintext.go b/cmd/packet_capture_plaintext.go new file mode 100644 index 000000000..e7cde6985 --- /dev/null +++ b/cmd/packet_capture_plaintext.go @@ -0,0 +1,30 @@ +package cmd + +import ( + "sync/atomic" + "time" + + "github.com/netobserv/flowlogs-pipeline/pkg/config" +) + +var plaintextPacketID uint64 + +func nextPlaintextPacketID() uint64 { + return atomic.AddUint64(&plaintextPacketID, 1) +} + +func assignPlaintextPacketID(m *config.GenericMap) uint64 { + id := nextPlaintextPacketID() + (*m)["PacketID"] = id + return id +} + +func plaintextTimestamp(m config.GenericMap) time.Time { + if t, ok := m["TimeFlowStartMs"].(float64); ok && t > 0 { + return time.UnixMilli(int64(t)) + } + if t, ok := m["Time"].(float64); ok { + return time.Unix(int64(t), 0) + } + return time.Now() +} diff --git a/cmd/packet_capture_plaintext_test.go b/cmd/packet_capture_plaintext_test.go new file mode 100644 index 000000000..12d2bb97e --- /dev/null +++ b/cmd/packet_capture_plaintext_test.go @@ -0,0 +1,28 @@ +package cmd + +import ( + "testing" + + "github.com/netobserv/flowlogs-pipeline/pkg/config" +) + +func TestAssignPlaintextPacketID(t *testing.T) { + plaintextPacketID = 0 + m1 := config.GenericMap{"RecordType": "plaintext"} + id1 := assignPlaintextPacketID(&m1) + m2 := config.GenericMap{"RecordType": "plaintext"} + id2 := assignPlaintextPacketID(&m2) + if id1 != 1 || id2 != 2 { + t.Fatalf("expected sequential ids 1,2 got %d,%d", id1, id2) + } + if m1["PacketID"] != uint64(1) || m2["PacketID"] != uint64(2) { + t.Fatalf("unexpected PacketID on maps: %v %v", m1["PacketID"], m2["PacketID"]) + } +} + +func TestPlaintextTimestampUsesMillis(t *testing.T) { + ts := plaintextTimestamp(config.GenericMap{"TimeFlowStartMs": float64(1_700_000_000_123)}) + if ts.UnixMilli() != 1_700_000_000_123 { + t.Fatalf("unexpected timestamp %v", ts) + } +} diff --git a/cmd/plaintext_format.go b/cmd/plaintext_format.go new file mode 100644 index 000000000..226066d4e --- /dev/null +++ b/cmd/plaintext_format.go @@ -0,0 +1,580 @@ +package cmd + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "strings" + "unicode" + + "github.com/netobserv/flowlogs-pipeline/pkg/config" +) + +const maxPlaintextUnwrapDepth = 6 + +var plaintextJSONSkipKeys = map[string]struct{}{ + "RecordType": {}, "Direction": {}, "TLSSource": {}, "Protocol": {}, + "SSLType": {}, "Time": {}, "TimeFlowStartMs": {}, "Pid": {}, "Tgid": {}, + "SrcAddr": {}, "DstAddr": {}, "SrcPort": {}, "DstPort": {}, "PlaintextLen": {}, + "PacketID": {}, "PcapAnnotated": {}, +} + +var plaintextJSONTextKeys = []string{ + "text", "message", "msg", "body", "content", "data", "response", "result", "error", +} + +var embeddedExportJSONMarkers = [][]byte{ + []byte(`{"RecordType":"plaintext"`), + []byte(`{"Direction":"write"`), + []byte(`{"Direction":"read"`), +} + +// plaintextPayloadBytes returns TLS application bytes after peeling wrappers. +func plaintextPayloadBytes(m config.GenericMap) []byte { + var raw []byte + if pt := plaintextFieldString(m); pt != "" { + if decoded, err := base64.StdEncoding.DecodeString(pt); err == nil && len(decoded) > 0 { + raw = decoded + } + } + if len(raw) == 0 { + if preview, ok := m["PlaintextPreview"].(string); ok && preview != "" { + raw = []byte(preview) + } + } + if len(raw) == 0 { + return nil + } + return unwrapPayloadLayers(raw) +} + +func plaintextFieldString(m config.GenericMap) string { + v, ok := m["Plaintext"] + if !ok || v == nil { + return "" + } + if s, ok := v.(string); ok { + return s + } + return "" +} + +func unwrapPayloadLayers(data []byte) []byte { + for i := 0; i < maxPlaintextUnwrapDepth; i++ { + next := unwrapPayloadLayer(data) + if bytes.Equal(next, data) { + break + } + data = next + } + return data +} + +func unwrapPayloadLayer(data []byte) []byte { + if u := unwrapEmbeddedExportRecord(data); !bytes.Equal(u, data) { + return u + } + if embedded := extractEmbeddedExportJSON(data); len(embedded) > 0 { + return embedded + } + if stripped := stripLeadingApplicationPrefix(data); !bytes.Equal(stripped, data) { + return stripped + } + return data +} + +// stripLeadingApplicationPrefix removes a short binary prefix (e.g. OpenSSL ssl_type +// metadata leaked before the userspace buffer) when it is followed by HTTP or JSON text. +func stripLeadingApplicationPrefix(data []byte) []byte { + if len(data) == 0 || isApplicationTextStart(data) { + return data + } + const maxSkip = 8 + limit := maxSkip + if len(data)-1 < limit { + limit = len(data) - 1 + } + for skip := 1; skip <= limit; skip++ { + rest := data[skip:] + if isApplicationTextStart(rest) { + return rest + } + } + return data +} + +func isApplicationTextStart(data []byte) bool { + if len(data) == 0 { + return false + } + for _, marker := range applicationTextMarkers { + if bytes.HasPrefix(data, marker) { + return true + } + } + return false +} + +var applicationTextMarkers = [][]byte{ + []byte("GET "), + []byte("POST "), + []byte("PUT "), + []byte("HEAD "), + []byte("HTTP/"), + []byte("DELETE "), + []byte("OPTIONS "), + []byte("PATCH "), + []byte("CONNECT "), + []byte("TRACE "), + []byte("NETOBSERV-"), + []byte("{"), +} + +// extractEmbeddedExportJSON finds agent export/jsonl records inside gRPC or HTTP/2 frames. +func extractEmbeddedExportJSON(data []byte) []byte { + for _, marker := range embeddedExportJSONMarkers { + searchFrom := 0 + for { + idx := bytes.Index(data[searchFrom:], marker) + if idx < 0 { + break + } + idx += searchFrom + objBytes, ok := sliceJSONObject(data, idx) + if !ok { + searchFrom = idx + 1 + continue + } + if inner := unwrapExportJSONObject(objBytes); len(inner) > 0 && !bytes.Equal(inner, objBytes) { + return inner + } + searchFrom = idx + 1 + } + } + return nil +} + +func sliceJSONObject(data []byte, start int) ([]byte, bool) { + if start < 0 || start >= len(data) || data[start] != '{' { + return nil, false + } + depth := 0 + inString := false + escape := false + for i := start; i < len(data); i++ { + c := data[i] + if inString { + if escape { + escape = false + continue + } + switch c { + case '\\': + escape = true + case '"': + inString = false + } + continue + } + switch c { + case '"': + inString = true + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return data[start : i+1], true + } + } + } + return nil, false +} + +func unwrapExportJSONObject(objBytes []byte) []byte { + var obj map[string]json.RawMessage + if err := json.Unmarshal(objBytes, &obj); err != nil { + return nil + } + if !isExportRecordObject(obj) { + return nil + } + if pt := jsonRawString(obj["Plaintext"]); pt != "" { + if decoded, err := base64.StdEncoding.DecodeString(pt); err == nil && len(decoded) > 0 { + return decoded + } + } + if preview := jsonRawString(obj["PlaintextPreview"]); preview != "" { + return []byte(preview) + } + return nil +} + +func isExportRecordObject(obj map[string]json.RawMessage) bool { + if jsonRawString(obj["RecordType"]) == "plaintext" { + return true + } + if _, hasPT := obj["Plaintext"]; hasPT { + if _, hasDir := obj["Direction"]; hasDir { + return true + } + } + return false +} + +// unwrapEmbeddedExportRecord handles a top-level jsonl/export record payload. +func unwrapEmbeddedExportRecord(data []byte) []byte { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || trimmed[0] != '{' { + return data + } + if inner := unwrapExportJSONObject(trimmed); len(inner) > 0 { + return inner + } + return data +} + +// formatPlaintextPayload turns captured TLS bytes into human-readable text for the TUI. +func formatPlaintextPayload(data []byte) string { + data = unwrapPayloadLayers(data) + for i := 0; i < 3; i++ { + next := extractHTTPBody(data) + next = extractHTTPRequestLine(next) + next = extractJSONTextBody(next) + if bytes.Equal(next, data) { + break + } + data = next + } + if isNoisePayload(data) { + return "" + } + display := plaintextDisplayString(data) + if isGarbageDisplay(display) { + return "" + } + return display +} + +// isMeaningfulPlaintextRecord reports whether a plaintext row is worth showing in the TUI. +func isMeaningfulPlaintextRecord(m config.GenericMap) bool { + return plaintextPreviewForDisplay(m) != "" +} + +// enrichPlaintextForExport adds a human-readable PlaintextDisplay field for jsonl consumers. +func enrichPlaintextForExport(m *config.GenericMap) { + if m == nil { + return + } + display := plaintextPreviewForDisplay(*m) + if display != "" { + (*m)["PlaintextDisplay"] = display + } +} + +func extractHTTPBody(data []byte) []byte { + if len(data) == 0 { + return data + } + if bytes.HasPrefix(data, []byte("HTTP/")) { + if idx := bytes.Index(data, []byte("\r\n\r\n")); idx >= 0 { + return data[idx+4:] + } + if idx := bytes.Index(data, []byte("\n\n")); idx >= 0 { + return data[idx+2:] + } + } + return data +} + +func extractHTTPRequestLine(data []byte) []byte { + if len(data) == 0 { + return data + } + switch data[0] { + case 'G', 'P', 'H', 'D', 'C', 'O', 'T': + default: + return data + } + if idx := bytes.IndexByte(data, '\n'); idx > 0 { + line := bytes.TrimSpace(data[:idx]) + if len(line) > 0 && bytes.IndexByte(line, ' ') > 0 { + return line + } + } + if idx := bytes.Index(data, []byte("\r\n")); idx > 0 { + return bytes.TrimSpace(data[:idx]) + } + return data +} + +func extractJSONTextBody(data []byte) []byte { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 { + return data + } + if trimmed[0] == '"' { + var s string + if err := json.Unmarshal(trimmed, &s); err == nil { + return []byte(s) + } + return data + } + if trimmed[0] != '{' { + return data + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(trimmed, &obj); err != nil { + return data + } + if isExportRecordObject(obj) { + return data + } + for _, key := range plaintextJSONTextKeys { + if s := jsonRawString(obj[key]); s != "" && !looksLikeStructuredJSON(s) { + return []byte(s) + } + } + best := "" + bestScore := -1 + for key, raw := range obj { + if _, skip := plaintextJSONSkipKeys[key]; skip { + continue + } + s := jsonRawString(raw) + if s == "" || looksLikeStructuredJSON(s) || looksLikeBase64Blob(s) { + continue + } + score := printableTextScore(s) + if score > bestScore { + bestScore = score + best = s + } + } + if best != "" { + return []byte(best) + } + return data +} + +func isNoisePayload(data []byte) bool { + if len(data) == 0 { + return true + } + if isTLSRecord(data) { + return true + } + if bytes.HasPrefix(data, []byte("PRI * HTTP/2")) { + return true + } + if bytes.Contains(data, []byte("\x1b[")) { + return true + } + if containsAgentExportLeak(data) { + return true + } + if len(data) <= 8 && printableRatio(data) < 0.5 { + return true + } + if bytes.HasPrefix(data, []byte{0x00, 0x00}) && printableRatio(data) < 0.72 { + return true + } + if printableRatio(data) < 0.35 && len(data) > 16 { + return true + } + trimmed := bytes.TrimSpace(data) + if len(trimmed) > 0 && trimmed[0] == '{' { + var obj map[string]json.RawMessage + if err := json.Unmarshal(trimmed, &obj); err == nil { + if isExportRecordObject(obj) || isPacketExportObject(obj) { + return true + } + } + } + return false +} + +func isTLSRecord(data []byte) bool { + if len(data) < 3 || data[1] != 0x03 { + return false + } + switch data[0] { + case 0x14, 0x15, 0x16, 0x17: + return data[2] == 0x01 || data[2] == 0x03 || data[2] == 0x04 + default: + return false + } +} + +func containsAgentExportLeak(data []byte) bool { + if bytes.Contains(data, []byte(`"RecordType":"plaintext"`)) { + return true + } + if bytes.Contains(data, []byte(`"Plaintext"`)) && bytes.Contains(data, []byte(`"Direction"`)) { + return true + } + if bytes.Contains(data, []byte(`"Bytes"`)) && bytes.Contains(data, []byte(`"Data"`)) { + return true + } + return false +} + +func isPacketExportObject(obj map[string]json.RawMessage) bool { + _, hasBytes := obj["Bytes"] + _, hasData := obj["Data"] + return hasBytes && hasData +} + +func plaintextDisplayString(data []byte) string { + s := strings.ToValidUTF8(string(data), "\uFFFD") + var b strings.Builder + for _, r := range s { + switch r { + case '\n', '\r', '\t': + b.WriteRune(r) + default: + if r >= 32 && r < 127 || r > 127 { + b.WriteRune(r) + } else { + fmt.Fprintf(&b, "\\x%02x", r) + } + } + } + return b.String() +} + +func isGarbageDisplay(s string) bool { + if s == "" { + return true + } + if strings.Contains(s, `\x`) { + return true + } + if strings.Contains(s, `{"Bytes":`) || strings.Contains(s, `{"Direction":`) { + return true + } + if strings.Contains(s, "\x1b[") { + return true + } + if printableRatio([]byte(s)) < 0.72 && len(s) > 24 { + return true + } + if len(strings.TrimSpace(s)) < 6 && !looksLikeShortPlaintext(s) { + return true + } + return false +} + +var shortPlaintextHTTPPrefixes = []string{ + "HTTP/", "GET ", "POST ", "PUT ", "DELETE ", "HEAD ", "OPTIONS ", "PATCH ", +} + +func looksLikeShortPlaintext(s string) bool { + trimmed := strings.TrimSpace(s) + if trimmed == "" { + return false + } + for _, prefix := range shortPlaintextHTTPPrefixes { + if strings.HasPrefix(trimmed, prefix) { + return true + } + } + if trimmed[0] == '{' || trimmed[0] == '[' { + return true + } + return isPlainIdentifierText(trimmed) && len(trimmed) >= 2 +} + +func isPlainIdentifierText(s string) bool { + for _, r := range s { + if r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '/' || r == '.' || r == '-' || r == '_' { + continue + } + return false + } + return true +} + +func printableRatio(data []byte) float64 { + if len(data) == 0 { + return 0 + } + printable := 0 + for _, b := range data { + if b == '\t' || b == '\n' || b == '\r' || (b >= 32 && b < 127) { + printable++ + } + } + return float64(printable) / float64(len(data)) +} + +func jsonRawString(raw json.RawMessage) string { + if len(raw) == 0 { + return "" + } + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s + } + return "" +} + +func looksLikeStructuredJSON(s string) bool { + t := strings.TrimSpace(s) + return strings.HasPrefix(t, "{") || strings.HasPrefix(t, "[") +} + +func looksLikeBase64Blob(s string) bool { + if len(s) < 32 { + return false + } + for _, r := range s { + switch { + case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '+', r == '/', r == '=': + default: + return false + } + } + _, err := base64.StdEncoding.DecodeString(s) + return err == nil +} + +func printableTextScore(s string) int { + if s == "" { + return -1 + } + score := len(s) + for _, r := range s { + if r == '\n' || r == '\r' || r == '\t' { + continue + } + if r < 32 || !unicode.IsPrint(r) { + score -= 4 + } + } + return score +} + +func plaintextPreviewForDisplay(m config.GenericMap, maxLen ...int) string { + payload := plaintextPayloadBytes(m) + if len(payload) == 0 { + return "" + } + display := formatPlaintextPayload(payload) + if display == "" { + return "" + } + limit := 120 + if len(maxLen) > 0 && maxLen[0] > 0 { + limit = maxLen[0] + } + return ellipsizePlaintextDisplay(display, limit) +} + +func ellipsizePlaintextDisplay(s string, maxLen int) string { + if maxLen <= 0 || len(s) <= maxLen { + return s + } + return fmt.Sprintf("%s...", s[:maxLen]) +} diff --git a/cmd/plaintext_format_test.go b/cmd/plaintext_format_test.go new file mode 100644 index 000000000..6bac3d402 --- /dev/null +++ b/cmd/plaintext_format_test.go @@ -0,0 +1,193 @@ +package cmd + +import ( + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/netobserv/flowlogs-pipeline/pkg/config" + "github.com/stretchr/testify/assert" +) + +func TestPlaintextPayloadBytesPrefersFullPlaintext(t *testing.T) { + full := []byte("HTTP/1.1 200 OK\r\n\r\nktls-test-pod response body") + m := config.GenericMap{ + "Plaintext": base64.StdEncoding.EncodeToString(full), + "PlaintextPreview": "HTTP/1.1 200 OK", + "PlaintextLen": float64(len(full)), + } + assert.Equal(t, full, plaintextPayloadBytes(m)) +} + +func TestFormatPlaintextPayloadHTTPResponse(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nktls-test-pod path=/api/items") + assert.Equal(t, "ktls-test-pod path=/api/items", formatPlaintextPayload(raw)) +} + +func TestFormatPlaintextPayloadHealthzResponse(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nDate: Thu, 25 Jun 2026 16:27:10 GMT\r\nContent-Length: 2\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nOK") + assert.Equal(t, "OK", formatPlaintextPayload(raw)) +} + +func TestFormatPlaintextPayloadJSONMessageField(t *testing.T) { + raw := []byte(`{"message":"hello from api","status":"ok"}`) + assert.Equal(t, "hello from api", formatPlaintextPayload(raw)) +} + +func TestFormatPlaintextPayloadHTTPRequestLine(t *testing.T) { + raw := []byte("GET /health HTTP/1.1\r\nHost: :8080\r\nUser-Agent: Go-http-client/1.1\r\n\r\n") + assert.Equal(t, "GET /health HTTP/1.1", formatPlaintextPayload(raw)) +} + +func TestFormatPlaintextPayloadStripsOpenSSLMetadataPrefix(t *testing.T) { + // Observed on openssl captures: ssl_type/metadata before decrypted HTTP (2026-06-26T132343Z.jsonl). + request := append([]byte{0x05, 0x00, 0x00, 0x00}, []byte("GET /healthz HTTP/1.1\r\nHost: 10.244.2.24:8443\r\n\r\n")...) + assert.Equal(t, "GET /healthz HTTP/1.1", formatPlaintextPayload(request)) + + response := append([]byte{0x05, 0x00, 0x00, 0x00}, []byte("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nContent-Type: text/plain\r\n\r\nOK")...) + assert.Equal(t, "OK", formatPlaintextPayload(response)) +} + +func TestPlaintextPayloadBytesStripsOpenSSLMetadataPrefix(t *testing.T) { + raw := append([]byte{0x05, 0x00, 0x00, 0x00}, []byte("NETOBSERV-OPENSSL probe")...) + m := config.GenericMap{ + "Plaintext": base64.StdEncoding.EncodeToString(raw), + } + assert.Equal(t, []byte("NETOBSERV-OPENSSL probe"), plaintextPayloadBytes(m)) +} + +func TestFormatPlaintextPayloadFromCaptureJSONL132343Z(t *testing.T) { + jsonl := filepath.Join("..", "output", "plaintext", "2026-06-26T132343Z.jsonl") + if _, err := os.Stat(jsonl); err != nil { + t.Skip("capture jsonl not present:", err) + } + data, err := os.ReadFile(jsonl) + if err != nil { + t.Fatal(err) + } + var readLine, writeLine string + for _, line := range splitJSONLLines(data) { + if line == "" { + continue + } + var m config.GenericMap + if err := json.Unmarshal([]byte(line), &m); err != nil { + t.Fatal(err) + } + switch m["Direction"] { + case "read": + if readLine == "" { + readLine = line + } + case "write": + if writeLine == "" { + writeLine = line + } + } + } + var readMap, writeMap config.GenericMap + if err := json.Unmarshal([]byte(readLine), &readMap); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(writeLine), &writeMap); err != nil { + t.Fatal(err) + } + assert.Equal(t, "GET /healthz HTTP/1.1", plaintextPreviewForDisplay(readMap)) + // Full write capture ends mid-headers (no body after \r\n\r\n); status line is still readable. + assert.Equal(t, "HTTP/1.1 200 OK", plaintextPreviewForDisplay(writeMap)) +} + +func TestFormatPlaintextPayloadUnwrapsExportRecordJSON(t *testing.T) { + body := []byte("ktls-test-pod path=/api/items method=GET") + export := config.GenericMap{ + "RecordType": "plaintext", + "Plaintext": base64.StdEncoding.EncodeToString(body), + "PlaintextPreview": string(body), + "Direction": "write", + "TLSSource": "ktls", + } + line, err := json.Marshal(export) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, string(body), formatPlaintextPayload(line)) +} + +func TestFormatPlaintextPayloadUnwrapsGRPCEmbeddedExportJSON(t *testing.T) { + inner := []byte("GET /readyz HTTP/1.1\r\nHost: 127.0.0.1:2381\r\n\r\n") + export := config.GenericMap{ + "Direction": "write", + "Plaintext": base64.StdEncoding.EncodeToString(inner), + "PlaintextPreview": string(inner), + "RecordType": "plaintext", + } + exportJSON, err := json.Marshal(export) + if err != nil { + t.Fatal(err) + } + // gRPC DATA-like prefix observed in 2026-06-25T162701Z.jsonl + wrapped := append([]byte("\x00\x00\x08\x01\x04\x00\x00\x00\x03\xff\xff\xff\xff\xff\xff\xff\x00\x00\x80\x00\x01\x00\x00\x00\x03\x00\x00\x00\x00\x80\n\x99\x01\x12\x99\x01"), exportJSON...) + assert.Equal(t, "GET /readyz HTTP/1.1", formatPlaintextPayload(wrapped)) +} + +func TestFormatPlaintextPayloadNoiseTLSRecord(t *testing.T) { + raw := []byte{0x17, 0x03, 0x03, 0x00, 0x19, 0x4b, 0xfb, 0x72} + assert.Equal(t, "", formatPlaintextPayload(raw)) +} + +func TestFormatPlaintextPayloadNoiseTinyBinary(t *testing.T) { + raw := []byte{0x00, 0x00, 0x00, 0x03} + assert.Equal(t, "", formatPlaintextPayload(raw)) +} + +func TestIsMeaningfulPlaintextRecordFromCaptureJSONL(t *testing.T) { + jsonl := filepath.Join("..", "output", "plaintext", "2026-06-25T162701Z.jsonl") + if _, err := os.Stat(jsonl); err != nil { + t.Skip("capture jsonl not present:", err) + } + data, err := os.ReadFile(jsonl) + if err != nil { + t.Fatal(err) + } + + meaningful := 0 + noise := 0 + for _, line := range splitJSONLLines(data) { + if line == "" { + continue + } + var m config.GenericMap + if err := json.Unmarshal([]byte(line), &m); err != nil { + t.Fatal(err) + } + if isMeaningfulPlaintextRecord(m) { + meaningful++ + } else { + noise++ + } + } + assert.Equal(t, 2479, meaningful+noise) + // 2026-06-25T162701Z.jsonl: kube/health probe HTTP lines; rest is kTLS noise + // (gRPC export feedback, TLS records, binary framing). Prefix stripping surfaces + // a few more openssl healthz rows that previously looked binary. + assert.Equal(t, 90, meaningful) + assert.Equal(t, 2389, noise) +} + +func splitJSONLLines(data []byte) []string { + var lines []string + start := 0 + for i := 0; i < len(data); i++ { + if data[i] != '\n' { + continue + } + lines = append(lines, string(data[start:i])) + start = i + 1 + } + if start < len(data) { + lines = append(lines, string(data[start:])) + } + return lines +} diff --git a/cmd/root.go b/cmd/root.go index e13812566..7a5f62aaf 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -107,7 +107,7 @@ func onInit() { printBanner() log.Infof("Log level: %s\nOption(s): %s", logLevel, options) - if strings.Contains(options, "background") && !strings.Contains(options, "background=false") { + if optionEnabled("background") { isBackground = true log.Infof("Running in background mode") } @@ -191,6 +191,15 @@ func onLimitReached() bool { return shouldExit } +// optionEnabled reports whether a named CLI option is set in the pipe-separated --options string. +// The shell passes flags as --name or name=true. +func optionEnabled(name string) bool { + if strings.Contains(options, name+"=false") { + return false + } + return strings.Contains(options, name+"=true") || strings.Contains(options, name) +} + // Create output file, preventing path traversal func createOutputFile(kind, filename string) (*os.File, error) { base := "./output/" + kind + "/" diff --git a/cmd/root_test.go b/cmd/root_test.go index 6c20fa10c..c8694904b 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -75,14 +75,43 @@ func TestDefaultArguments(t *testing.T) { assert.Empty(t, options) } +func TestOptionEnabled(t *testing.T) { + options = "port=443|--enable_openssl" + assert.True(t, optionEnabled("enable_openssl")) + assert.False(t, optionEnabled("enable_gotls")) + + options = "enable_openssl=true|port=443" + assert.True(t, optionEnabled("enable_openssl")) + + options = "enable_openssl=false" + assert.False(t, optionEnabled("enable_openssl")) +} + +func TestPlaintextCaptureEnabled(t *testing.T) { + options = "--enable_openssl" + assert.True(t, plaintextCaptureEnabled()) + + options = "enable_gotls=true" + assert.False(t, plaintextCaptureEnabled()) + + options = "port=443" + assert.False(t, plaintextCaptureEnabled()) +} + func setup(t *testing.T) { // reset time to startup time resetTime() + capture = Flow + options = "" + // clear filters and previous flows regexes = []string{} lastFlows = []config.GenericMap{} + clearPacketCaptureBuffers() showCount = defaultFlowShowCount + selectedData = []byte{} + paused = false // clear previous table content tableData = &TableData{ diff --git a/docs/tls-decryption-coverage.md b/docs/tls-decryption-coverage.md new file mode 100644 index 000000000..b1d93de27 --- /dev/null +++ b/docs/tls-decryption-coverage.md @@ -0,0 +1,135 @@ +# TLS Decryption Coverage Matrix + +Reference for NetObserv CLI on-demand packet capture with TLS plaintext visibility on OpenShift. + +## Approaches + +| Approach | Coverage on OpenShift | Requires app changes | CLI flag | Agent env | +|----------|----------------------|----------------------|----------|-----------| +| PCA wire capture | Cleartext HTTP only | No | (default in packets mode) | `ENABLE_PCA=true` | +| OpenSSL uprobes | Apps using libssl (nginx, curl, Python, Ruby, PHP) | No | `--enable_openssl` | `ENABLE_OPENSSL_TRACKING=true` | +| GoTLS uprobes | Go binaries (`crypto/tls`) | No | `--enable_gotls` | `ENABLE_GOTLS_TRACKING=true` (write path only by default) | +| kTLS sk_msg | Kernel TLS offload (nginx+kTLS, niche) | No | `--enable_ktls` | `ENABLE_KTLS_TRACKING=true` | + +Test workloads: [openssl-test-pod](../examples/openssl-test-pod/) (`--enable_openssl`), [gotls-test-pod](../examples/gotls-test-pod/) (`--enable_gotls`), [ktls-test-pod](../examples/ktls-test-pod/) (`--enable_ktls`). +| SSLKEYLOGFILE + pcapng DSB | Any TLS when customer sets env var | Yes | `--tls-keylog` | (none) | + +## OpenShift scenarios + +| Scenario | Wire PCAP | Recommended path | +|----------|-----------|------------------| +| HTTP on port 80 | Readable | PCA | +| Route terminates TLS, pod sees HTTP | Readable | PCA | +| Pod HTTPS (OpenSSL) | Encrypted | OpenSSL uprobes (`--enable_openssl`); test workload: [examples/openssl-test-pod](../examples/openssl-test-pod/) | +| Go microservice | Encrypted | GoTLS uprobes (`--enable_gotls`) | +| Mixed OpenSSL + Go workloads | Encrypted | Both flags together (`--enable_openssl --enable_gotls`) | +| Service mesh mTLS (Istio/CSM) | Encrypted | BoringSSL on Envoy (future) | +| Customer can set `SSLKEYLOGFILE` | Encrypted | CLI `--tls-keylog` | +| nginx with kTLS | Encrypted on wire | kTLS tracking (`--enable_ktls`); test workload: [examples/ktls-test-pod](../examples/ktls-test-pod/) | + +## CLI outputs + +| File | Path | Contents | +|------|------|----------| +| Wire capture | `./output/pcap/.pcapng` | Encrypted TLS on the wire; when a plaintext event matches a buffered frame (5-tuple + time), `PlaintextPreview` is appended as an EPB comment on that **real** packet | +| Plaintext sidecar | `./output/plaintext/.jsonl` | One JSON object per TLS plaintext event (when TLS flags enabled) | + +Plaintext JSONL fields: `RecordType`, `PacketID`, `PcapAnnotated`, `Time`, `TimeFlowStartMs`, `Pid`, `Tgid`, `Direction`, `TLSSource`, `Plaintext` (base64), `PlaintextLen`, `PlaintextPreview`, `SSLType`, and when available `SrcAddr`, `DstAddr`, `SrcPort`, `DstPort`, `Protocol`. + +Deploy a collector image that includes the TLS correlation code. The default `quay.io/netobserv/network-observability-cli:main` image does not yet ship this feature; build locally and set `NETOBSERV_COLLECTOR_IMAGE` before running `oc netobserv packets`. + +`PcapAnnotated` is `true` when the CLI matched the event to a wire packet (strict 5-tuple + time). When multiple workloads share a capture port, port-only correlation is refused; use `--peer_ip` or rely on agent socket-fd 5-tuple enrichment. + +## Flow filters vs plaintext + +`--port`, `--peer_ip`, `--peer_cidr`, and other `FLOW_FILTER_RULES` apply to **pcapng wire packets** (PCA / TC hook). + +For plaintext, the agent applies matching rules via `PlaintextScope`: + +| Filter | Wire (pcapng) | Plaintext JSONL | Uprobe discovery | +|--------|---------------|-----------------|------------------| +| `--peer_ip` / `--peer_cidr` | Yes | Yes (PID scope + 5-tuple match) | Yes (limits which PIDs are hooked) | +| `--port` | Yes | Yes (when 5-tuple enriched) | No (OpenSSL hooks per libssl path, not per-PID) | +| Port-only, no peer IP | Yes | Yes (port match on enriched 5-tuple) | No | + +PID scope refreshes every 5s by scanning `/proc//net/tcp` for sockets involving the peer IP/CIDR. Override with `TLS_PLAINTEXT_PID_ALLOWLIST` (comma-separated PIDs). + +Duplicate plaintext events (same PID, direction, payload prefix) are dropped within `TLS_PLAINTEXT_DEDUP_WINDOW` (default 500ms). + +Set `TLS_PLAINTEXT_MIN_BYTES` on the agent (default `0`) to drop short TLS fragments before export. For on-demand CLI captures, `4` or `8` removes most WebSocket/binary framing noise (~18–37% of events in typical router HTTPS traffic) while keeping readable HTTP payloads. + +CLI: `--tls_plaintext_min_bytes=4` (packets mode only) sets the agent env on the capture DaemonSet. + +`PlaintextPreview` length defaults to 256 bytes. Set `--tls_plaintext_preview_bytes=0` for the full captured payload in preview (max 16 KiB per event), or another positive value to customize. The TUI also decodes the full base64 `Plaintext` field when the preview is shorter than `PlaintextLen`. + +## OpenShift deployment requirements + +TLS plaintext capture requires elevated privileges on the agent DaemonSet: + +- `--privileged` or `SYS_PTRACE` capability +- `hostPID: true` on the **pod template** (`spec.template.spec.hostPID`) for per-process library discovery +- Host `/proc` access for libssl path scanning + +Agent mounts host `/usr`, `/lib`, `/lib64` under `/host/` when using `--enable_openssl` so uprobes can open discovered `libssl.so` paths. Container workloads are hooked via `/proc//root/.../libssl.so` (same path in maps, different inode than host libssl). + +## Limitations + +- **5-tuple enrichment** — OpenSSL/GoTLS resolve the socket **fd** (`SSL_set_fd` map + `SSL*` BIO fallback; GoTLS `*tls.Conn` walk), map fd→inode via `/proc//fd`, then match `/proc/net/tcp`. kTLS carries the kernel 5-tuple in the eBPF event. Falls back to netns IP + connection affinity when fd is unknown. +- **No K8s pod metadata on plaintext** — FLP enrichment keys off `SrcAddr`/`DstAddr`; pod/namespace labels require future enrichment once 5-tuple is reliable +- Java JSSE (non-OpenSSL) is not covered by OpenSSL uprobes +- Statically linked OpenSSL may need explicit library path via `OPENSSL_PATH` +- GoTLS auto-discovers Go executables from `/proc/*/exe` and resolves `writeRecordLocked` / `Read` offsets from `.gopclntab` (Go 1.17+ register ABI) +- GoTLS auto-discovery skips node infrastructure binaries (kubelet, crio, ovnkube, multus, console, host `/usr/bin/kube-*`, `/usr/bin/openshift-*`); OpenSSL discovery uses the same skip list and only attaches per-container `libssl.so` under `/proc//root` (never the host default `OPENSSL_PATH`, which would hook every process on the node). **Hard-denied** node/CNI binaries (kubelet, crio, multus, ovnkube, …) are never hooked, even with `--peer_ip`. **Soft-excluded** workloads (e.g. openshift-console) are skipped in broad scans but allowed when `--peer_ip` / `--peer_cidr` scopes discovery to that PID. +- Recommended with TLS plaintext flags: `--peer_ip=` (or `--peer_cidr`) **and** `--port=`. They solve different problems: + - **`--peer_ip` / `--peer_cidr`**: scopes **which processes get uprobes** (OpenSSL libssl per container, GoTLS binary discovery, kTLS PID allowlist). Without peer scope the CLI warns; the agent hooks broader targets on the node. + - **`--port`**: filters **exported plaintext and wire capture** to matching src/dst ports; also helps 5-tuple enrichment when addresses are partial. It does **not** reduce uprobe attachment — only narrows what you see in the TUI/JSONL/pcap. +- Optional GoTLS overrides: `GOTLS_ELF_PATH`, `GOTLS_WRITE_OFFSET`, `GOTLS_READ_OFFSET` +- kTLS: IPv6 supported in agent; cgroup attach required on RHEL CoreOS nodes; niche workloads only +- PCA wire capture truncates frames to 256 bytes; plaintext uses a separate 16 KiB ringbuf per event + +## Troubleshooting empty plaintext JSONL + +1. Confirm capture command includes `--enable_openssl --privileged` (sets `hostPID`, host lib mounts, `ENABLE_OPENSSL_TRACKING`). +2. Check agent logs for `attached SSL_write uprobe to /proc//root/.../libssl.so` — host-only `/host/usr/...` hooks system processes, not pod containers. +3. Look for `TLS plaintext event captured` in agent logs during HTTPS traffic. +4. TLS on the wire (visible in pcapng) is not enough: decryption requires in-process OpenSSL in the pod under capture on that node. +5. Go (`crypto/tls`) — use `--enable_gotls --privileged` (auto-discovery). Java apps need SSLKEYLOGFILE. +6. With `--peer_ip`, confirm the target pod has an established TCP socket to that IP (PID scope is refreshed every 5s). + +### GoTLS setup + +`--enable_gotls` enables auto-discovery: the agent scans `/proc` for Go binaries, parses `.gopclntab`, and attaches a uprobe to `crypto/tls.(*Conn).writeRecordLocked` (outbound application data — HTTP responses on TLS servers). **Read/inbound capture** (`GOTLS_CAPTURE_READ`) is off by default because uprobes on `(*Conn).Read` crash Go targets; do not enable unless you accept that risk. + +Optional overrides when auto-discovery fails (stripped/custom builds): + +| Env var | Purpose | +|---------|---------| +| `GOTLS_ELF_PATH` | Pin a single Go binary path | +| `GOTLS_WRITE_OFFSET` | File offset for write hook | +| `GOTLS_READ_OFFSET` | File offset for a single Read RET hook | + +Requires `hostPID: true` (set automatically with `--enable_gotls --privileged`). **Recommended:** `--peer_ip=` and `--port=`; unscoped peer discovery hooks all non-excluded Go binaries on the node. + +## MVP status (validated) + +- OpenSSL plaintext on OpenShift console traffic (HTTPS + WebSocket JSON) via `--enable_openssl` +- Output: pcapng + `output/plaintext/.jsonl` +- TUI shows `PlaintextPreview` when a plaintext record is selected + +Live packet TUI behavior with plaintext capture (`--enable_openssl`, `--enable_gotls`, and/or `--enable_ktls`): + +- The table lists **TLS plaintext rows only** (wire packets stay in the pcapng); green rows are meaningful plaintext (HTTP/JSON-like payloads) +- When multiple TLS sources are active, the table keeps the newest row per source (`openssl`, `gotls`, `ktls`) visible alongside recent events +- **Event / Type** shows the TLS source for plaintext rows; use **PlaintextPreview** for the decoded payload +- Pause the capture, then click a green row to open the **TLS Plaintext** text panel +- Press Esc to resume live capture +- P0–P3: wall-clock timestamps, PID scoping, userspace 5-tuple enrichment, deduplication, wire↔plaintext correlation (`PcapAnnotated`, pcapng comments) + +## Remaining work + +| Priority | Item | Why | +|----------|------|-----| +| P4 | BoringSSL / Envoy (service mesh) | Phase 5 in plan | +| P5 | kTLS validation on OpenShift | Phase 4 in plan | +| — | ~~eBPF 5-tuple on `ssl_data_event_t`~~ | kTLS kernel tuple + fd→inode for OpenSSL/GoTLS (done) | +| — | K8s pod/namespace metadata on plaintext | TUI join with workload identity | diff --git a/examples/Makefile b/examples/Makefile new file mode 100644 index 000000000..1315fa0bc --- /dev/null +++ b/examples/Makefile @@ -0,0 +1,85 @@ +# Orchestrate OpenSSL TLS and cleartext HTTP test workloads (NETOBSERV-2858). +VERSION ?= latest +IMAGE_ORG ?= $(USER) +PULL_POLICY ?= Always + +K8S_CLI_BIN_PATH := $(shell which oc 2>/dev/null || which kubectl) +K8S_CLI_BIN ?= $(shell basename ${K8S_CLI_BIN_PATH}) + +TLS_STACKS := openssl-test-pod +HTTP_STACK := http-test-pod +ALL_STACKS := $(TLS_STACKS) $(HTTP_STACK) + +.PHONY: help +help: ## Show targets + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Build + +.PHONY: images-all +images-all: ## Build and push all test images (OpenSSL TLS + HTTP) + @for d in $(ALL_STACKS); do $(MAKE) -C $$d images VERSION=$(VERSION) IMAGE_ORG=$(IMAGE_ORG); done + +.PHONY: images-http +images-http: ## Build and push cleartext HTTP test image only + @$(MAKE) -C $(HTTP_STACK) images VERSION=$(VERSION) IMAGE_ORG=$(IMAGE_ORG) + +.PHONY: deploy-local-all +deploy-local-all: ## Kind: load images then deploy all stacks locally + @for d in $(ALL_STACKS); do $(MAKE) -C $$d deploy-local VERSION=$(VERSION) IMAGE_ORG=$(IMAGE_ORG); done + +.PHONY: deploy-local-http +deploy-local-http: ## Kind: load and deploy cleartext HTTP pod + @$(MAKE) -C $(HTTP_STACK) deploy-local VERSION=$(VERSION) IMAGE_ORG=$(IMAGE_ORG) + +##@ Cluster + +.PHONY: deploy-all +deploy-all: ## Deploy all test pods (OpenSSL TLS + HTTP) + @for d in $(ALL_STACKS); do $(MAKE) -C $$d deploy VERSION=$(VERSION) IMAGE_ORG=$(IMAGE_ORG) PULL_POLICY=$(PULL_POLICY); done + +.PHONY: deploy-http +deploy-http: ## Deploy cleartext HTTP test pod + @$(MAKE) -C $(HTTP_STACK) deploy VERSION=$(VERSION) IMAGE_ORG=$(IMAGE_ORG) PULL_POLICY=$(PULL_POLICY) + +.PHONY: wait-all +wait-all: ## Wait until all deployments are Available + @for d in $(ALL_STACKS); do $(MAKE) -C $$d wait; done + +.PHONY: wait-http +wait-http: ## Wait for HTTP test deployment + @$(MAKE) -C $(HTTP_STACK) wait + +.PHONY: traffic-all +traffic-all: ## Start traffic Jobs for all stacks + @for d in $(ALL_STACKS); do $(MAKE) -C $$d traffic; done + +.PHONY: traffic-http +traffic-http: ## Start cleartext HTTP traffic Job + @$(MAKE) -C $(HTTP_STACK) traffic + +.PHONY: pod-ips +pod-ips: ## Print pod IPs for --peer_ip scoping + @echo "OPENSSL_POD_IP=$$($(MAKE) -sC openssl-test-pod pod-ip)" + @echo "HTTP_POD_IP=$$($(MAKE) -sC http-test-pod pod-ip)" + +.PHONY: capture-hint +capture-hint: pod-ips ## Print example netobserv packets commands + @eval "$$($(MAKE) -s pod-ips | sed 's/^/export /')" + @echo "" + @echo "# Cleartext HTTP (PCA only — no TLS flags):" + @echo './build/oc-netobserv packets --port=8080 --peer_ip="$$HTTP_POD_IP" --background' + @echo "" + @echo "# OpenSSL TLS plaintext JSONL (use agent image from NETOBSERV-2857):" + @echo 'NETOBSERV_AGENT_IMAGE= ./build/oc-netobserv packets --port=8443 --peer_ip="$$OPENSSL_POD_IP" --enable_openssl --privileged --background' + @echo 'jq -r "select(.RecordType==\"plaintext\") | {TLSSource,PlaintextDisplay}" output/plaintext/*.jsonl | head' + @echo "" + @echo "# Live TUI + wire/pcap correlation: NETOBSERV-2859" + +.PHONY: undeploy-all +undeploy-all: ## Remove all test namespaces + @for d in $(ALL_STACKS); do $(MAKE) -C $$d undeploy; done + +.PHONY: undeploy-http +undeploy-http: ## Remove HTTP test namespace + @$(MAKE) -C $(HTTP_STACK) undeploy diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 000000000..0e1464ee4 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,115 @@ +# TLS plaintext showcase workloads + +Three minimal HTTPS pods demo NetObserv TLS plaintext capture. A fourth **cleartext HTTP** pod demos PCA wire readability (no TLS flags). + +| Pod | TLS stack | CLI flag | Marker | Example header | +|-----|-----------|----------|--------|----------------| +| [http-test-pod](http-test-pod/) | none (HTTP) | _(PCA only)_ | `NETOBSERV-HTTP` | `X-NetObserv-Stack: http` | +| [openssl-test-pod](openssl-test-pod/) | nginx + `libssl.so` | `--enable_openssl` | `NETOBSERV-OPENSSL` | `X-NetObserv-Stack: openssl` | +| [gotls-test-pod](gotls-test-pod/) | Go `crypto/tls` | `--enable_gotls` | `NETOBSERV-GOTLS` | `X-NetObserv-Stack: gotls` | +| [ktls-test-pod](ktls-test-pod/) | nginx + kTLS (`sk_msg`) | `--enable_ktls` | `NETOBSERV-KTLS` | `X-NetObserv-Stack: ktls` | + +## Cleartext HTTP (PCA) + +```bash +cd examples/http-test-pod && make deploy-local wait traffic +export HTTP_POD_IP=$(make pod-ip) +./build/oc-netobserv packets --port=8080 --peer_ip="${HTTP_POD_IP}" +``` + +Select a wire row — the detail panel shows **Wire HTTP (cleartext)** when the TCP payload is HTTP (status line + headers + body). TLS pods still use the green **TLS Plaintext** panel. + +## Quick start (TLS stacks) + +```bash +cd examples + +# Build, deploy, wait, generate traffic +make images-all IMAGE_ORG=$USER +make deploy-all wait-all traffic-all + +# Pod IPs for scoped capture +make pod-ips +make capture-hint # prints ready-to-paste commands +``` + +Kind without a registry: + +```bash +make deploy-local-all wait-all traffic-all +``` + +## What each endpoint showcases + +| Endpoint | Purpose | +|----------|---------| +| `GET /message` | Multi-line plaintext + fake response headers (`HTTP/1.1 200 OK`, `X-Fake-Trace-Id`, …) | +| `GET /api/items` | JSON body with stack-specific `"sku"` values | +| `POST /api/echo` | POST body from traffic Job; **gotls** also echoes `received_request_headers` in the response | +| `GET /healthz` | Short probe response (kubelet noise — keep for readiness only) | + +Traffic Jobs send fake **request** headers (`Authorization`, `X-Fake-Trace-Id: -req-*`, `X-NetObserv-Client`) on every call. + +## Capture commands + +**Recommended:** one stack + `--peer_ip` (less noise, reliable 5-tuple): + +```bash +export OPENSSL_POD_IP=$(make -sC openssl-test-pod pod-ip) + +NETOBSERV_AGENT_IMAGE=quay.io/you/netobserv-ebpf-agent:plaintext \ + ./build/oc-netobserv packets \ + --port=8443 \ + --peer_ip="${OPENSSL_POD_IP}" \ + --enable_openssl \ + --privileged +``` + +**Stress test:** all flags, port only (no `--peer_ip`): + +```bash +./build/oc-netobserv packets --port=8443 --enable_openssl --enable_gotls --enable_ktls --privileged +``` + +Annotated plaintext (`PcapAnnotated: true`) picks up `SrcK8S_*` / `DstK8S_*` from the correlated wire packet (PCA rows are FLP-enriched). For better agent 5-tuple without a single pod IP, use `--peer_cidr=10.244.0.0/16` (Kind default pod network). See `make capture-hint`. + +Keep `make traffic-all` running during capture. + +## What to look for in the live TUI + +- **Green rows** — TLS plaintext (`RecordType` column shows `openssl` / `gotls` / `ktls`) +- **`PlaintextPreview`** — should include `HTTP/1.1 200 OK`, fake `X-Fake-*` headers, then `NETOBSERV- …` +- **Binary noise** — kTLS/gRPC framing shows as `` when not HTTP-shaped +- **Row select** — opens hex panel with full decoded payload (pauses live ingest) +- **`+` / `-`** — increase “Showing last: N” if the table feels sparse + +## Verify JSONL output + +```bash +# Count by stack +jq -r '.TLSSource' output/plaintext/*.jsonl | sort | uniq -c + +# Distinct markers in previews +rg 'NETOBSERV-(OPENSSL|GOTLS|KTLS)' output/plaintext/ + +# Wire ↔ plaintext correlation +jq 'select(.PcapAnnotated == true) | {TLSSource, PlaintextPreview}' output/plaintext/*.jsonl | head +``` + +## Optional CLI knobs to demo + +| Flag | Showcase | +|------|----------| +| `--tls_plaintext_min_bytes=4` | Drop tiny TLS fragments; cleaner table, fewer binary rows | +| `--tls_plaintext_preview_bytes=512` | Longer `PlaintextPreview` column (includes more headers) | +| `--peer_cidr=10.244.0.0/16` | Scope to pod network instead of single `--peer_ip` | + +See [docs/tls-decryption-coverage.md](../docs/tls-decryption-coverage.md) for limits, `PcapAnnotated`, and fd→inode 5-tuple enrichment. + +## Ideas not covered yet (future) + +- **Large response** (`/large`, 8–16 KiB) — preview truncation vs hex panel +- **HTTP errors** (`418` / `404`) — non-200 status lines in plaintext +- **Read-direction capture** — inbound HTTP requests on server (GoTLS read uprobes disabled today) +- **SSLKEYLOGFILE** sidecar — `--tls-keylog` wire decryption path (requires app change) +- **Java / BoringSSL** workloads — Envoy/Istio mesh scenarios diff --git a/examples/http-test-pod/Dockerfile b/examples/http-test-pod/Dockerfile new file mode 100644 index 000000000..921a28cc7 --- /dev/null +++ b/examples/http-test-pod/Dockerfile @@ -0,0 +1,21 @@ +FROM docker.io/fedora:40 + +RUN dnf install -y nginx \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +COPY nginx.conf /etc/nginx/nginx.conf +COPY entrypoint.sh /entrypoint.sh +COPY html/healthz /var/www/html/healthz +COPY html/index.html /var/www/html/index.html +COPY html/message /var/www/html/message +COPY html/api/items /var/www/html/api/items +COPY html/api/echo /var/www/html/api/echo + +RUN mkdir -p /tmp/nginx/{client_body,proxy,fastcgi,uwsgi,scgi} /var/www/html/api \ + && chmod +x /entrypoint.sh \ + && chmod -R a+rwX /tmp/nginx /var/www/html /var/log/nginx /var/lib/nginx + +USER 65534:0 +EXPOSE 8080 +ENTRYPOINT ["/entrypoint.sh"] diff --git a/examples/http-test-pod/Makefile b/examples/http-test-pod/Makefile new file mode 100644 index 000000000..a1564e5cf --- /dev/null +++ b/examples/http-test-pod/Makefile @@ -0,0 +1,75 @@ +# note: to build and push custom image tag use: IMAGE_ORG=netobserv VERSION=latest +VERSION ?= latest + +GOARCH ?= amd64 + +IMAGE_ORG ?= $(USER) + +NAME := http-test +NAMESPACE ?= http-test + +K8S_CLI_BIN_PATH := $(shell which oc 2>/dev/null || which kubectl) +K8S_CLI_BIN ?= $(shell basename ${K8S_CLI_BIN_PATH}) + +IMAGE_REGISTRY ?= quay.io + +IMAGE_TAG_BASE ?= $(IMAGE_REGISTRY)/$(IMAGE_ORG)/$(NAME) +IMAGE ?= $(IMAGE_TAG_BASE):$(VERSION) +PULL_POLICY ?= Always + +OCI_BIN_PATH := $(shell which docker 2>/dev/null || which podman) +OCI_BIN ?= $(shell basename ${OCI_BIN_PATH}) +OCI_BUILD_OPTS ?= + +ifeq ("$(OCI_BIN)","docker") +EXTRA_BUILD_FLAGS ?= --provenance=false +endif + +KIND_CLUSTER_NAME ?= netobserv-cli-cluster + +.PHONY: image-build +image-build: + DOCKER_BUILDKIT=1 $(OCI_BIN) build $(OCI_BUILD_OPTS) $(EXTRA_BUILD_FLAGS) -t $(IMAGE) . + +.PHONY: image-push +image-push: + DOCKER_BUILDKIT=1 $(OCI_BIN) push $(IMAGE) + +.PHONY: images +images: image-build image-push + +.PHONY: kind-load +kind-load: + kind load docker-image $(IMAGE) --name $(KIND_CLUSTER_NAME) + +.PHONY: deploy-local +deploy-local: + $(MAKE) kind-load + $(MAKE) deploy PULL_POLICY=IfNotPresent + +.PHONY: deploy +deploy: + sed -e 's|__HTTP_TEST_IMAGE__|$(IMAGE)|g' \ + -e 's|__IMAGE_PULL_POLICY__|$(PULL_POLICY)|g' \ + deployment.yaml | $(K8S_CLI_BIN) apply -f - + +.PHONY: undeploy +undeploy: + $(K8S_CLI_BIN) delete -f deployment.yaml --ignore-not-found + $(K8S_CLI_BIN) delete -f traffic-job.yaml --ignore-not-found + +.PHONY: traffic +traffic: + $(K8S_CLI_BIN) apply -f traffic-job.yaml + +.PHONY: wait +wait: + $(K8S_CLI_BIN) wait -n $(NAMESPACE) deployment/http-test --for=condition=Available --timeout=120s + +.PHONY: pod-ip +pod-ip: + @$(K8S_CLI_BIN) get pod -n $(NAMESPACE) -l app=http-test -o jsonpath='{.items[0].status.podIP}{"\n"}' + +.PHONY: help +help: + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/examples/http-test-pod/README.md b/examples/http-test-pod/README.md new file mode 100644 index 000000000..17c7fbe90 --- /dev/null +++ b/examples/http-test-pod/README.md @@ -0,0 +1,42 @@ +# HTTP cleartext test pod + +Minimal pod that serves **plain HTTP** (no TLS) on port **8080**. Use it to validate NetObserv PCA wire capture: HTTP is readable on the wire without `--enable_openssl`, `--enable_gotls`, or `--enable_ktls`. + +## Deploy and traffic + +```bash +cd examples/http-test-pod +make images IMAGE_ORG=$USER # or deploy-local on Kind +make deploy wait traffic +export HTTP_POD_IP=$(make pod-ip) +``` + +## Capture + +No TLS flags — PCA only: + +```bash +./build/oc-netobserv packets \ + --port=8080 \ + --peer_ip="${HTTP_POD_IP}" \ + --max-bytes=100000000 +``` + +## Live TUI + +- **White wire rows** — encrypted-looking traffic is absent; TCP payloads carry HTTP. +- **Select a row** — when the packet contains cleartext HTTP, the detail panel shows **Wire HTTP (cleartext)** with status line, headers, and body (not hex). +- Non-HTTP wire packets still open the hex view. + +Marker in responses: `NETOBSERV-HTTP` (see `GET /message`). + +## Compare with TLS examples + +| Pod | Port | Capture flags | Detail panel | +|-----|------|---------------|--------------| +| **http-test** (this) | 8080 | none (PCA) | Wire HTTP (cleartext) | +| [openssl-test](../openssl-test-pod/) | 8443 | `--enable_openssl` | TLS Plaintext | +| [gotls-test](../gotls-test-pod/) | 8443 | `--enable_gotls` | TLS Plaintext | +| [ktls-test](../ktls-test-pod/) | 8443 | `--enable_ktls` | TLS Plaintext | + +See [examples/README.md](../README.md) for the full TLS showcase. diff --git a/examples/http-test-pod/deployment.yaml b/examples/http-test-pod/deployment.yaml new file mode 100644 index 000000000..2becd3f14 --- /dev/null +++ b/examples/http-test-pod/deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: http-test +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: http-test + namespace: http-test + labels: + app: http-test +spec: + replicas: 1 + selector: + matchLabels: + app: http-test + template: + metadata: + labels: + app: http-test + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + hostUsers: false + fsGroup: 0 + containers: + - name: server + image: __HTTP_TEST_IMAGE__ + imagePullPolicy: __IMAGE_PULL_POLICY__ + ports: + - name: http + containerPort: 8080 + readinessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: ["ALL"] +--- +apiVersion: v1 +kind: Service +metadata: + name: http-test + namespace: http-test +spec: + selector: + app: http-test + ports: + - name: http + port: 8080 + targetPort: http diff --git a/examples/http-test-pod/entrypoint.sh b/examples/http-test-pod/entrypoint.sh new file mode 100644 index 000000000..56dd9e89e --- /dev/null +++ b/examples/http-test-pod/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -euo pipefail + +echo "nginx: $(nginx -v 2>&1)" + +if ! nginx -t 2>&1; then + echo "nginx config test failed" >&2 + exit 1 +fi + +exec nginx -g 'daemon off;' diff --git a/examples/http-test-pod/html/api/echo b/examples/http-test-pod/html/api/echo new file mode 100644 index 000000000..ecf3eb9ec --- /dev/null +++ b/examples/http-test-pod/html/api/echo @@ -0,0 +1,4 @@ +NETOBSERV-HTTP POST echo response +Transport: plain HTTP (no TLS) +Workload: http-test-pod +Endpoint: POST /api/echo diff --git a/examples/http-test-pod/html/api/items b/examples/http-test-pod/html/api/items new file mode 100644 index 000000000..c2f76a2df --- /dev/null +++ b/examples/http-test-pod/html/api/items @@ -0,0 +1 @@ +{"source":"http","message":"NETOBSERV-HTTP api/items","items":[{"id":1,"sku":"http-alpha"},{"id":2,"sku":"http-beta"}],"stack":"nginx cleartext"} diff --git a/examples/http-test-pod/html/healthz b/examples/http-test-pod/html/healthz new file mode 100644 index 000000000..9766475a4 --- /dev/null +++ b/examples/http-test-pod/html/healthz @@ -0,0 +1 @@ +ok diff --git a/examples/http-test-pod/html/index.html b/examples/http-test-pod/html/index.html new file mode 100644 index 000000000..56b5ba076 --- /dev/null +++ b/examples/http-test-pod/html/index.html @@ -0,0 +1,4 @@ +NETOBSERV-HTTP cleartext probe +Transport: plain HTTP (no TLS) +Workload: http-test-pod +Endpoint: GET / diff --git a/examples/http-test-pod/html/message b/examples/http-test-pod/html/message new file mode 100644 index 000000000..193a273f6 --- /dev/null +++ b/examples/http-test-pod/html/message @@ -0,0 +1,5 @@ +NETOBSERV-HTTP cleartext probe +Transport: plain HTTP (no TLS) +Workload: http-test-pod +Endpoint: GET /message +Capture hint: readable in PCA wire packets on port 8080 — select row for HTTP text panel diff --git a/examples/http-test-pod/nginx.conf b/examples/http-test-pod/nginx.conf new file mode 100644 index 000000000..8aa1096b7 --- /dev/null +++ b/examples/http-test-pod/nginx.conf @@ -0,0 +1,66 @@ +worker_processes 1; +pid /tmp/nginx.pid; +error_log /dev/stderr info; + +events { + worker_connections 1024; +} + +http { + access_log /dev/stdout; + sendfile on; + + client_body_temp_path /tmp/nginx/client_body; + proxy_temp_path /tmp/nginx/proxy; + fastcgi_temp_path /tmp/nginx/fastcgi; + uwsgi_temp_path /tmp/nginx/uwsgi; + scgi_temp_path /tmp/nginx/scgi; + + server { + listen 8080; + server_name http-test; + + root /var/www/html; + + location /healthz { + default_type text/plain; + try_files /healthz =404; + } + + location = /message { + default_type text/plain; + add_header X-NetObserv-Stack http always; + add_header X-NetObserv-Endpoint message always; + add_header X-Fake-Authorization "Bearer fake-http-demo-token" always; + add_header X-Fake-Trace-Id http-resp-message always; + add_header X-Fake-Tenant demo-http always; + add_header Cache-Control "no-store" always; + try_files /message =404; + } + + location = /api/items { + default_type application/json; + add_header X-NetObserv-Stack http always; + add_header X-NetObserv-Endpoint api-items always; + add_header X-Fake-Authorization "Bearer fake-http-demo-token" always; + add_header X-Fake-Trace-Id http-resp-items always; + add_header Cache-Control "no-store" always; + try_files /api/items =404; + } + + location = /api/echo { + default_type text/plain; + add_header X-NetObserv-Stack http always; + add_header X-NetObserv-Endpoint api-echo always; + add_header X-Fake-Trace-Id http-resp-echo always; + try_files /api/echo =404; + } + + location / { + default_type text/plain; + add_header X-NetObserv-Stack http always; + add_header X-NetObserv-Endpoint root always; + try_files $uri /index.html; + } + } +} diff --git a/examples/http-test-pod/traffic-job.yaml b/examples/http-test-pod/traffic-job.yaml new file mode 100644 index 000000000..c8570df20 --- /dev/null +++ b/examples/http-test-pod/traffic-job.yaml @@ -0,0 +1,52 @@ +# Generates steady cleartext HTTP traffic to the http-test server (nginx, no TLS). +apiVersion: batch/v1 +kind: Job +metadata: + name: http-test-traffic + namespace: http-test +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: curl + image: docker.io/curlimages/curl:8.11.1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + command: + - /bin/sh + - -c + - | + set -e + TARGET="${TARGET_URL:-http://http-test.http-test.svc.cluster.local:8080/}" + echo "Traffic to ${TARGET}" + i=0 + while [ "$i" -lt 120 ]; do + curl --http1.1 --no-keepalive -s \ + -H 'Authorization: Bearer fake-http-client-token' \ + -H "X-Fake-Trace-Id: http-req-message-${i}" \ + -H 'X-Fake-Tenant: demo-http' \ + -H 'X-NetObserv-Client: http-traffic-pod' \ + "${TARGET}message?seq=${i}" || true + curl --http1.1 --no-keepalive -s \ + -H 'Authorization: Bearer fake-http-client-token' \ + -H "X-Fake-Trace-Id: http-req-items-${i}" \ + -H 'X-NetObserv-Client: http-traffic-pod' \ + "${TARGET}api/items?seq=${i}" || true + curl --http1.1 --no-keepalive -s -X POST \ + -H 'Authorization: Bearer fake-http-client-token' \ + -H "X-Fake-Trace-Id: http-req-echo-${i}" \ + -H 'X-NetObserv-Client: http-traffic-pod' \ + -H 'Content-Type: text/plain' \ + -d "NETOBSERV-HTTP client request seq=${i}" \ + "${TARGET}api/echo?seq=${i}" || true + i=$((i + 1)) + sleep 1 + done + echo "done" diff --git a/examples/openssl-test-pod/Dockerfile b/examples/openssl-test-pod/Dockerfile new file mode 100644 index 000000000..236593245 --- /dev/null +++ b/examples/openssl-test-pod/Dockerfile @@ -0,0 +1,28 @@ +FROM docker.io/fedora:40 + +RUN dnf install -y nginx openssl \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +COPY nginx.conf /etc/nginx/nginx.conf +COPY entrypoint.sh /entrypoint.sh +COPY html/healthz /var/www/html/healthz +COPY html/index.html /var/www/html/index.html +COPY html/message /var/www/html/message +COPY html/api/items /var/www/html/api/items +COPY html/api/echo /var/www/html/api/echo + +# OpenShift/Kubernetes may run an arbitrary UID without group 0 — use world-readable +# paths for this self-signed test cert and nginx temp dirs. +RUN mkdir -p /tmp/nginx/{client_body,proxy,fastcgi,uwsgi,scgi} /tmp/tls /var/www/html/api \ + && openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout /tmp/tls/key.pem -out /tmp/tls/cert.pem \ + -days 365 -subj "/CN=openssl-test-pod" \ + -addext "subjectAltName=DNS:openssl-test,DNS:localhost" \ + && chmod +x /entrypoint.sh \ + && chmod -R a+rwX /tmp/nginx /tmp/tls /var/www/html /var/log/nginx /var/lib/nginx \ + && chmod a+r /tmp/tls/cert.pem /tmp/tls/key.pem + +USER 65534:0 +EXPOSE 8443 +ENTRYPOINT ["/entrypoint.sh"] diff --git a/examples/openssl-test-pod/Makefile b/examples/openssl-test-pod/Makefile new file mode 100644 index 000000000..68f103107 --- /dev/null +++ b/examples/openssl-test-pod/Makefile @@ -0,0 +1,80 @@ +# note: to build and push custom image tag use: IMAGE_ORG=netobserv VERSION=latest +VERSION ?= latest + +GOARCH ?= amd64 + +# In CI, to be replaced by `netobserv` +IMAGE_ORG ?= $(USER) + +NAME := openssl-test +NAMESPACE ?= openssl-test + +K8S_CLI_BIN_PATH := $(shell which oc 2>/dev/null || which kubectl) +K8S_CLI_BIN ?= $(shell basename ${K8S_CLI_BIN_PATH}) + +IMAGE_REGISTRY ?= quay.io + +IMAGE_TAG_BASE ?= $(IMAGE_REGISTRY)/$(IMAGE_ORG)/$(NAME) +IMAGE ?= $(IMAGE_TAG_BASE):$(VERSION) +PULL_POLICY ?= Always + +OCI_BIN_PATH := $(shell which docker 2>/dev/null || which podman) +OCI_BIN ?= $(shell basename ${OCI_BIN_PATH}) +OCI_BUILD_OPTS ?= + +ifeq ("$(OCI_BIN)","docker") +EXTRA_BUILD_FLAGS ?= --provenance=false +endif + +KIND_CLUSTER_NAME ?= netobserv-cli-cluster + +##@ Images + +.PHONY: image-build +image-build: ## Build container image + DOCKER_BUILDKIT=1 $(OCI_BIN) build $(OCI_BUILD_OPTS) $(EXTRA_BUILD_FLAGS) -t $(IMAGE) . + +.PHONY: image-push +image-push: ## Push container image + DOCKER_BUILDKIT=1 $(OCI_BIN) push $(IMAGE) + +.PHONY: images +images: image-build image-push ## Build and push image + +.PHONY: kind-load +kind-load: ## Load $(IMAGE) into Kind (no registry pull needed) + kind load docker-image $(IMAGE) --name $(KIND_CLUSTER_NAME) + +.PHONY: deploy-local +deploy-local: ## Kind: load image then deploy with IfNotPresent + $(MAKE) kind-load + $(MAKE) deploy PULL_POLICY=IfNotPresent + +##@ Cluster + +.PHONY: deploy +deploy: ## Apply manifests using $(IMAGE) + sed -e 's|__OPENSSL_TEST_IMAGE__|$(IMAGE)|g' \ + -e 's|__IMAGE_PULL_POLICY__|$(PULL_POLICY)|g' \ + deployment.yaml | $(K8S_CLI_BIN) apply -f - + +.PHONY: undeploy +undeploy: ## Delete test namespace and traffic job + $(K8S_CLI_BIN) delete -f deployment.yaml --ignore-not-found + $(K8S_CLI_BIN) delete -f traffic-job.yaml --ignore-not-found + +.PHONY: traffic +traffic: ## Start curl Job against the service + $(K8S_CLI_BIN) apply -f traffic-job.yaml + +.PHONY: wait +wait: ## Wait for deployment Available + $(K8S_CLI_BIN) wait -n $(NAMESPACE) deployment/openssl-test --for=condition=Available --timeout=120s + +.PHONY: pod-ip +pod-ip: ## Print pod IP for --peer_ip + @$(K8S_CLI_BIN) get pod -n $(NAMESPACE) -l app=openssl-test -o jsonpath='{.items[0].status.podIP}{"\n"}' + +.PHONY: help +help: ## Display this help + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/examples/openssl-test-pod/README.md b/examples/openssl-test-pod/README.md new file mode 100644 index 000000000..64522dfd7 --- /dev/null +++ b/examples/openssl-test-pod/README.md @@ -0,0 +1,154 @@ +# OpenSSL test pod + +Minimal pod that serves HTTPS with **nginx + dynamically linked OpenSSL 3** (`libssl.so`). Use it to validate NetObserv packet capture with `--enable_openssl`. + +This is the right workload for OpenSSL uprobes. It does **not** use Go `crypto/tls` (see [gotls-test-pod](../gotls-test-pod/)) and does **not** enable kernel TLS offload (see [ktls-test-pod](../ktls-test-pod/)). + +## Why not use gotls-test or ktls-test with `--enable_openssl`? + +| Test pod | TLS stack | Flag for green plaintext rows | +|----------|-----------|-------------------------------| +| **openssl-test** (this) | nginx + `libssl.so` | `--enable_openssl` | +| [gotls-test](../gotls-test-pod/) | Go `crypto/tls` | `--enable_gotls` (not `--enable_openssl`) | +| [ktls-test](../ktls-test-pod/) | nginx + kTLS offload | `--enable_ktls` when kernel offloads; if kTLS is active, OpenSSL uprobes often see **no** application data | + +## 1. Build and push (Quay) + +Default image: `quay.io/$(USER)/openssl-test:latest` + +```bash +cd examples/openssl-test-pod + +podman login quay.io # or docker login + +make images + +make images IMAGE_ORG=netobserv +# => quay.io/netobserv/openssl-test:latest +``` + +**Local Kind cluster** (no registry pull): + +```bash +make image-build USER=jpinsonn VERSION=test +make deploy-local USER=jpinsonn VERSION=test +``` + +## 2. Deploy + +```bash +make deploy IMAGE_ORG=netobserv +make wait +export OPENSSL_POD_IP=$(make pod-ip) +echo "Pod IP: $OPENSSL_POD_IP" +``` + +Verify nginx links libssl dynamically: + +```bash +kubectl logs -n openssl-test deploy/openssl-test | grep libssl +# expect: libssl (nginx): libssl.so.3 => /lib64/libssl.so.3 +``` + +## 3. Generate HTTPS traffic + +```bash +make traffic +# or one-off: +kubectl run -n openssl-test curl-once --rm -it --restart=Never \ + --image=curlimages/curl:8.11.1 --command -- \ + curl --http1.1 -sk "https://${OPENSSL_POD_IP}:8443/message" +``` + +Expected response body (each workload uses a unique `NETOBSERV-*` marker): + +```text +NETOBSERV-OPENSSL plaintext probe +TLS stack: nginx + OpenSSL 3 (userspace libssl.so) +Workload: openssl-test-pod +Endpoint: GET /message +Capture hint: look for "NETOBSERV-OPENSSL" in PlaintextPreview +``` + +`make traffic` also hits `/api/items` (JSON with `"sku":"openssl-alpha"`) and `POST /api/echo` with a client body `NETOBSERV-OPENSSL client request seq=N`. + +Fake HTTP headers are included on purpose (not real credentials): + +- **Response** (visible in server `SSL_write` plaintext): `X-NetObserv-Stack`, `X-Fake-Authorization`, `X-Fake-Trace-Id`, … +- **Request** (sent by the traffic Job): `Authorization`, `X-Fake-Trace-Id: openssl-req-*`, `X-NetObserv-Client: openssl-traffic-pod` + +In the live TUI, `PlaintextPreview` should show lines like `HTTP/1.1 200 OK` and `X-Fake-Trace-Id: openssl-resp-message` ahead of the body. + +## 4. Capture with NetObserv + +Agent image must include OpenSSL uprobe support (`ENABLE_OPENSSL_TRACKING`). + +```bash +export OPENSSL_POD_IP=$(make pod-ip) +NETOBSERV_AGENT_IMAGE=quay.io/jpinsonn/netobserv-ebpf-agent:plaintext3 \ +NETOBSERV_COLLECTOR_IMAGE=quay.io/jpinsonn/network-observability-cli:plaintext2 \ + ./build/oc-netobserv packets \ + --port=8443 \ + --peer_ip="${OPENSSL_POD_IP}" \ + --enable_openssl \ + --privileged \ + --max-bytes=100000000 +``` + +`--peer_ip` scopes uprobes and helps plaintext pass agent filters when the 5-tuple is not enriched yet. Match **both** agent and collector image tags to your local builds (`plaintext3` / `plaintext2` above are examples). + +`--enable_openssl` sets `hostPID`, host `/usr`/`/lib`/`/lib64` mounts, and `ENABLE_OPENSSL_TRACKING=true` on the agent DaemonSet. + +Keep traffic running during capture (`make traffic` or repeated `curl`). + +### What to expect + +- **Green TUI rows** with HTTP response bodies containing `NETOBSERV-OPENSSL` +- **JSONL** (`output/plaintext/*.jsonl`): `"TLSSource": "openssl"`, `"Direction": "write"` for server responses +- **Agent logs** on the node running the pod: + +```bash +kubectl logs -n netobserv-cli -l app=netobserv-cli --tail=200 | rg -i 'openssl|SSL_write|plaintext' +``` + +Look for `attached SSL_write uprobe` and `TLS plaintext event captured` with `source=openssl`. + +`--peer_ip` is optional for OpenSSL discovery but recommended to reduce noise from other node processes. + +## Makefile reference + +| Variable | Default | Example | +|----------|---------|---------| +| `IMAGE_REGISTRY` | `quay.io` | `docker.io` | +| `IMAGE_ORG` | `$(USER)` | `netobserv` | +| `VERSION` | `latest` | `test` | +| `IMAGE` | `$(IMAGE_REGISTRY)/$(IMAGE_ORG)/openssl-test:$(VERSION)` | full override | +| `PULL_POLICY` | `Always` | `IfNotPresent` (local) | + +Run `make help` for targets. + +## Troubleshooting + +| Symptom | Check | +|---------|--------| +| CLI warns about missing `--peer_ip` | Expected without peer scope — add `--peer_ip=$(make pod-ip)` to narrow hooks and reduce noise | +| No green rows with `--enable_openssl` on **gotls-test** | Wrong flag — use `--enable_gotls` for Go TLS | +| No green rows on **ktls-test** with `--enable_openssl` | kTLS may offload TLS to the kernel; use this **openssl-test** pod, or `--enable_ktls` on ktls-test | +| No green rows, agent logs show `TLS plaintext event captured` but JSONL is empty | **`--port` without `--peer_ip`** used to drop plaintext before export when no 5-tuple (fixed in agent `plaintext3+`); workaround: add `--peer_ip=`; rebuild agent | +| Green rows delayed ~30s | Plaintext waits for wire-packet correlation; pause capture or wait for buffer flush | +| `no libssl.so libraries discovered` in agent logs | Agent needs `hostPID=true` and `--privileged`; redeploy capture with `--enable_openssl --privileged` | +| `attached SSL_write` but no plaintext | Custom agent image with TLS capture; keep `make traffic` running; check `FLOW_FILTER_RULES` port matches 8443 | +| `peer_ip` scoped but no hooks | `hostPID` must be true; pod IP must match `--peer_ip`; agent DaemonSet on the **same node** as the workload | +| Only health-probe lines (`GET /healthz`) | Normal — also hit `/` and `/api/items` via `make traffic` | +| `ImagePullBackOff` | `make images` + public Quay repo or pull secret in `openssl-test` namespace | + +## Cleanup + +```bash +make undeploy +``` + +## Related examples + +- [GoTLS test pod](../gotls-test-pod/) — `--enable_gotls` +- [kTLS test pod](../ktls-test-pod/) — `--enable_ktls` diff --git a/examples/openssl-test-pod/deployment.yaml b/examples/openssl-test-pod/deployment.yaml new file mode 100644 index 000000000..82a9e77e1 --- /dev/null +++ b/examples/openssl-test-pod/deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openssl-test +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openssl-test + namespace: openssl-test + labels: + app: openssl-test +spec: + replicas: 1 + selector: + matchLabels: + app: openssl-test + template: + metadata: + labels: + app: openssl-test + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + hostUsers: false + fsGroup: 0 + containers: + - name: server + image: __OPENSSL_TEST_IMAGE__ + imagePullPolicy: __IMAGE_PULL_POLICY__ + ports: + - name: https + containerPort: 8443 + readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: ["ALL"] +--- +apiVersion: v1 +kind: Service +metadata: + name: openssl-test + namespace: openssl-test +spec: + selector: + app: openssl-test + ports: + - name: https + port: 8443 + targetPort: https diff --git a/examples/openssl-test-pod/entrypoint.sh b/examples/openssl-test-pod/entrypoint.sh new file mode 100644 index 000000000..db3fe341e --- /dev/null +++ b/examples/openssl-test-pod/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail + +echo "OpenSSL: $(openssl version)" +echo "nginx: $(nginx -v 2>&1)" +echo "libssl (nginx): $(ldd "$(command -v nginx)" 2>/dev/null | grep -E 'libssl\.so' || echo 'not found')" + +if ! nginx -t 2>&1; then + echo "nginx config test failed" >&2 + exit 1 +fi + +exec nginx -g 'daemon off;' diff --git a/examples/openssl-test-pod/html/api/echo b/examples/openssl-test-pod/html/api/echo new file mode 100644 index 000000000..08712db45 --- /dev/null +++ b/examples/openssl-test-pod/html/api/echo @@ -0,0 +1,5 @@ +NETOBSERV-OPENSSL POST echo response +TLS stack: nginx + OpenSSL 3 (userspace libssl.so) +Workload: openssl-test-pod +Endpoint: POST /api/echo +Note: static response body (OpenSSL SSL_write on nginx reply) diff --git a/examples/openssl-test-pod/html/api/items b/examples/openssl-test-pod/html/api/items new file mode 100644 index 000000000..f3bd5c938 --- /dev/null +++ b/examples/openssl-test-pod/html/api/items @@ -0,0 +1 @@ +{"source":"openssl","message":"NETOBSERV-OPENSSL api/items","items":[{"id":1,"sku":"openssl-alpha"},{"id":2,"sku":"openssl-beta"}],"stack":"nginx+libssl.so userspace"} diff --git a/examples/openssl-test-pod/html/healthz b/examples/openssl-test-pod/html/healthz new file mode 100644 index 000000000..9766475a4 --- /dev/null +++ b/examples/openssl-test-pod/html/healthz @@ -0,0 +1 @@ +ok diff --git a/examples/openssl-test-pod/html/index.html b/examples/openssl-test-pod/html/index.html new file mode 100644 index 000000000..ad5fa713d --- /dev/null +++ b/examples/openssl-test-pod/html/index.html @@ -0,0 +1,4 @@ +NETOBSERV-OPENSSL plaintext probe +TLS stack: nginx + OpenSSL 3 (userspace libssl.so) +Workload: openssl-test-pod +Endpoint: GET / diff --git a/examples/openssl-test-pod/html/message b/examples/openssl-test-pod/html/message new file mode 100644 index 000000000..c2db889f6 --- /dev/null +++ b/examples/openssl-test-pod/html/message @@ -0,0 +1,5 @@ +NETOBSERV-OPENSSL plaintext probe +TLS stack: nginx + OpenSSL 3 (userspace libssl.so) +Workload: openssl-test-pod +Endpoint: GET /message +Capture hint: look for "NETOBSERV-OPENSSL" in PlaintextPreview diff --git a/examples/openssl-test-pod/nginx.conf b/examples/openssl-test-pod/nginx.conf new file mode 100644 index 000000000..0cd62b8ab --- /dev/null +++ b/examples/openssl-test-pod/nginx.conf @@ -0,0 +1,78 @@ +worker_processes 1; +pid /tmp/nginx.pid; +error_log /dev/stderr info; + +events { + worker_connections 1024; +} + +http { + access_log /dev/stdout; + sendfile on; + + client_body_temp_path /tmp/nginx/client_body; + proxy_temp_path /tmp/nginx/proxy; + fastcgi_temp_path /tmp/nginx/fastcgi; + uwsgi_temp_path /tmp/nginx/uwsgi; + scgi_temp_path /tmp/nginx/scgi; + + server { + # HTTP/1.1 only — predictable SSL_write capture on response headers + bodies. + listen 8443 ssl; + server_name openssl-test; + + root /var/www/html; + + ssl_certificate /tmp/tls/cert.pem; + ssl_certificate_key /tmp/tls/key.pem; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + location /healthz { + default_type text/plain; + try_files /healthz =404; + } + + location = /message { + default_type text/plain; + add_header X-NetObserv-Stack openssl always; + add_header X-NetObserv-Endpoint message always; + add_header X-Fake-Authorization "Bearer fake-openssl-demo-token" always; + add_header X-Fake-Trace-Id openssl-resp-message always; + add_header X-Fake-Tenant demo-openssl always; + add_header X-Content-Type-Options nosniff always; + add_header Cache-Control "no-store" always; + try_files /message =404; + } + + location = /api/items { + default_type application/json; + add_header X-NetObserv-Stack openssl always; + add_header X-NetObserv-Endpoint api-items always; + add_header X-Fake-Authorization "Bearer fake-openssl-demo-token" always; + add_header X-Fake-Trace-Id openssl-resp-items always; + add_header X-Fake-Tenant demo-openssl always; + add_header Cache-Control "no-store" always; + try_files /api/items =404; + } + + location = /api/echo { + default_type text/plain; + add_header X-NetObserv-Stack openssl always; + add_header X-NetObserv-Endpoint api-echo always; + add_header X-Fake-Authorization "Bearer fake-openssl-demo-token" always; + add_header X-Fake-Trace-Id openssl-resp-echo always; + add_header X-Fake-Tenant demo-openssl always; + add_header Cache-Control "no-store" always; + try_files /api/echo =404; + } + + location / { + default_type text/plain; + add_header X-NetObserv-Stack openssl always; + add_header X-NetObserv-Endpoint root always; + add_header X-Fake-Trace-Id openssl-resp-root always; + try_files $uri /index.html; + } + } +} diff --git a/examples/openssl-test-pod/traffic-job.yaml b/examples/openssl-test-pod/traffic-job.yaml new file mode 100644 index 000000000..8b56a5085 --- /dev/null +++ b/examples/openssl-test-pod/traffic-job.yaml @@ -0,0 +1,54 @@ +# Generates steady HTTPS traffic to the OpenSSL test server (nginx + libssl.so). +apiVersion: batch/v1 +kind: Job +metadata: + name: openssl-test-traffic + namespace: openssl-test +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: curl + image: docker.io/curlimages/curl:8.11.1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + command: + - /bin/sh + - -c + - | + set -e + TARGET="${TARGET_URL:-https://openssl-test.openssl-test.svc.cluster.local:8443/}" + echo "Traffic to ${TARGET}" + i=0 + while [ "$i" -lt 120 ]; do + curl --http1.1 --no-keepalive -sk \ + -H 'Authorization: Bearer fake-openssl-client-token' \ + -H "X-Fake-Trace-Id: openssl-req-message-${i}" \ + -H 'X-Fake-Tenant: demo-openssl' \ + -H 'X-NetObserv-Client: openssl-traffic-pod' \ + "${TARGET}message?seq=${i}" || true + curl --http1.1 --no-keepalive -sk \ + -H 'Authorization: Bearer fake-openssl-client-token' \ + -H "X-Fake-Trace-Id: openssl-req-items-${i}" \ + -H 'X-Fake-Tenant: demo-openssl' \ + -H 'X-NetObserv-Client: openssl-traffic-pod' \ + "${TARGET}api/items?seq=${i}" || true + curl --http1.1 --no-keepalive -sk -X POST \ + -H 'Authorization: Bearer fake-openssl-client-token' \ + -H "X-Fake-Trace-Id: openssl-req-echo-${i}" \ + -H 'X-Fake-Tenant: demo-openssl' \ + -H 'X-NetObserv-Client: openssl-traffic-pod' \ + -H 'Content-Type: text/plain' \ + -d "NETOBSERV-OPENSSL client request seq=${i}" \ + "${TARGET}api/echo?seq=${i}" || true + i=$((i + 1)) + sleep 1 + done + echo "done" diff --git a/res/packet-capture.yml b/res/packet-capture.yml index 27d02b5af..8f80cbb93 100644 --- a/res/packet-capture.yml +++ b/res/packet-capture.yml @@ -16,6 +16,7 @@ spec: spec: serviceAccountName: netobserv-cli hostNetwork: true + hostPID: false dnsPolicy: ClusterFirstWithHostNet tolerations: - operator: Exists @@ -40,6 +41,12 @@ spec: value: "1s" - name: ENABLE_PCA value: "true" + - name: ENABLE_OPENSSL_TRACKING + value: "false" + - name: TLS_PLAINTEXT_MIN_BYTES + value: "0" + - name: TLS_PLAINTEXT_PREVIEW_BYTES + value: "256" - name: METRICS_ENABLE value: "false" - name: LOG_LEVEL diff --git a/scripts/functions.sh b/scripts/functions.sh index cd12179fa..b8905f437 100755 --- a/scripts/functions.sh +++ b/scripts/functions.sh @@ -621,6 +621,28 @@ function edit_manifest() { "ipsec_enable") "$YQ_BIN" e --inplace ".spec.template.spec.containers[0].env[] |= select(.name==\"ENABLE_IPSEC_TRACKING\").value|=\"$2\"" "$manifest" ;; + "openssl_enable") + "$YQ_BIN" e --inplace ".spec.template.spec.containers[0].env[] |= select(.name==\"ENABLE_OPENSSL_TRACKING\").value|=\"$2\"" "$manifest" + ;; + "tls_plaintext_min_bytes") + "$YQ_BIN" e --inplace ".spec.template.spec.containers[0].env[] |= select(.name==\"TLS_PLAINTEXT_MIN_BYTES\").value|=\"$2\"" "$manifest" + ;; + "tls_plaintext_preview_bytes") + "$YQ_BIN" e --inplace ".spec.template.spec.containers[0].env[] |= select(.name==\"TLS_PLAINTEXT_PREVIEW_BYTES\").value|=\"$2\"" "$manifest" + ;; + "tls_host_mounts") + if [[ "$("$YQ_BIN" e '[.spec.template.spec.volumes[] | select(.name == "host-usr")] | length' "$manifest")" == "0" ]]; then + "$YQ_BIN" e --inplace '.spec.template.spec.volumes += [{"name":"host-usr","hostPath":{"path":"/usr","type":"Directory"}},{"name":"host-lib","hostPath":{"path":"/lib","type":"Directory"}},{"name":"host-lib64","hostPath":{"path":"/lib64","type":"Directory"}}]' "$manifest" + "$YQ_BIN" e --inplace '.spec.template.spec.containers[0].volumeMounts += [{"name":"host-usr","mountPath":"/host/usr","readOnly":true},{"name":"host-lib","mountPath":"/host/lib","readOnly":true},{"name":"host-lib64","mountPath":"/host/lib64","readOnly":true}]' "$manifest" + fi + "$YQ_BIN" e --inplace '.spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem = false' "$manifest" + if [[ "$("$YQ_BIN" e '(.spec.template.spec.containers[0].securityContext.capabilities.add // []) | any(. == "SYS_PTRACE")' "$manifest")" != "true" ]]; then + "$YQ_BIN" e --inplace '.spec.template.spec.containers[0].securityContext.capabilities.add += ["SYS_PTRACE"]' "$manifest" + fi + ;; + "host_pid") + "$YQ_BIN" e --inplace ".spec.template.spec.hostPID|=$2" "$manifest" + ;; "privileged") "$YQ_BIN" e --inplace ".spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation|=$2" "$manifest" "$YQ_BIN" e --inplace ".spec.template.spec.containers[0].securityContext.privileged|=$2" "$manifest" @@ -837,6 +859,147 @@ function waitDaemonset(){ } # Validate options and edit manifest accordingly +function option_basename() { + local key="${1%%=*}" + echo "${key#--}" +} + +function option_enabled_flag() { + local option="$1" + local key="${option%%=*}" + local value="${option#*=}" + if [[ "$key" == "$value" ]]; then + return 0 + fi + [[ "$value" == "true" ]] +} + +function has_plaintext_pid_scope() { + for option in "${options[@]}"; do + case "$(option_basename "$option")" in + peer_ip|peer_cidr) + local value="${option#*=}" + if [[ -n "$value" && "${option%%=*}" != "$value" ]]; then + return 0 + fi + ;; + esac + done + return 1 +} + +function plaintext_capture_flag_enabled() { + local flag="$1" + for option in "${options[@]}"; do + case "$(option_basename "$option")" in + "$flag") + if option_enabled_flag "$option"; then + return 0 + fi + ;; + esac + done + return 1 +} + +function openssl_capture_enabled() { + plaintext_capture_flag_enabled enable_openssl +} + +function has_port_filter() { + for option in "${options[@]}"; do + case "$(option_basename "$option")" in + port|dport|sport|ports|dports|sports|port_range|dport_range|sport_range) + local value="${option#*=}" + if [[ -n "$value" && "${option%%=*}" != "$value" ]]; then + return 0 + fi + ;; + esac + done + return 1 +} + +function plaintext_port_filter_label() { + for option in "${options[@]}"; do + case "$(option_basename "$option")" in + port|dport|sport|ports|dports|sports|port_range|dport_range|sport_range) + local key + key="$(option_basename "$option")" + local value="${option#*=}" + if [[ -n "$value" && "${option%%=*}" != "$value" ]]; then + echo "${key}=${value}" + return + fi + ;; + esac + done +} + +function plaintext_peer_scope_label() { + for option in "${options[@]}"; do + case "$(option_basename "$option")" in + peer_ip) + local value="${option#*=}" + if [[ -n "$value" && "${option%%=*}" != "$value" ]]; then + echo "peer_ip=${value}" + return + fi + ;; + peer_cidr) + local value="${option#*=}" + if [[ -n "$value" && "${option%%=*}" != "$value" ]]; then + echo "peer_cidr=${value}" + return + fi + ;; + esac + done +} + +function warn_plaintext_peer_scope() { + if [[ "$command" != "packets" ]]; then + return + fi + local openssl=0 + plaintext_capture_flag_enabled enable_openssl && openssl=1 + if [[ $openssl -eq 0 ]]; then + return + fi + + local peer_scope=0 port_filter=0 + has_plaintext_pid_scope && peer_scope=1 + has_port_filter && port_filter=1 + + if [[ $peer_scope -eq 1 || $port_filter -eq 1 ]]; then + local scope_parts=() + if [[ $peer_scope -eq 1 ]]; then + scope_parts+=("$(plaintext_peer_scope_label)") + fi + if [[ $port_filter -eq 1 ]]; then + scope_parts+=("$(plaintext_port_filter_label)") + fi + local IFS=', ' + echo "Plaintext capture scoped to ${scope_parts[*]}." + fi + + if [[ $peer_scope -eq 0 ]]; then + echo >&2 + echo "Warning: TLS plaintext capture has no --peer_ip or --peer_cidr scope." >&2 + if [[ $openssl -eq 1 ]]; then + echo " --enable_openssl: hooks libssl.so in every container on each node (infra binaries excluded)." >&2 + fi + echo " Recommended: --peer_ip= to limit which processes are hooked." >&2 + echo >&2 + fi + + if [[ $port_filter -eq 0 ]]; then + echo "Warning: TLS plaintext capture has no --port (or --dport) filter." >&2 + echo " Recommended: --port= to filter exported plaintext and wire capture." >&2 + echo >&2 + fi +} + function parse_args() { # Iterate through the command-line arguments for option in "${options[@]}"; do @@ -1022,6 +1185,50 @@ function parse_args() { echo "invalid value for --enable_all" fi ;; + *enable_openssl) # OpenSSL plaintext capture via libssl uprobes + if [[ "$command" == "packets" ]]; then + defaultValue "true" + if [[ "$value" == "true" ]]; then + edit_manifest "privileged" "$value" + edit_manifest "host_pid" "$value" + edit_manifest "tls_host_mounts" "" + edit_manifest "openssl_enable" "$value" + elif [[ "$value" == "false" ]]; then + echo + else + echo "invalid value for --enable_openssl" + fi + else + echo "--enable_openssl is invalid option for $command" + exit 1 + fi + ;; + *tls_plaintext_min_bytes) # Drop short TLS plaintext events before agent export + if [[ "$command" == "packets" ]]; then + if [[ "$value" =~ ^[0-9]+$ ]]; then + edit_manifest "tls_plaintext_min_bytes" "$value" + else + echo "invalid value for --tls_plaintext_min_bytes (non-negative integer required)" + exit 1 + fi + else + echo "--tls_plaintext_min_bytes is invalid option for $command" + exit 1 + fi + ;; + *tls_plaintext_preview_bytes) # PlaintextPreview length on exported events + if [[ "$command" == "packets" ]]; then + if [[ "$value" =~ ^[0-9]+$ ]]; then + edit_manifest "tls_plaintext_preview_bytes" "$value" + else + echo "invalid value for --tls_plaintext_preview_bytes (non-negative integer required; 0 = full payload)" + exit 1 + fi + else + echo "--tls_plaintext_preview_bytes is invalid option for $command" + exit 1 + fi + ;; *privileged) # Force privileged mode defaultValue "true" if [[ "$value" == "true" ]]; then @@ -1177,6 +1384,8 @@ function parse_args() { esac done + warn_plaintext_peer_scope + # avoid packet capture without filters if [[ "$command" = "packets" ]]; then currentFilters=$("$YQ_BIN" -r ".spec.template.spec.containers[0].env[] | select(.name == \"FLOW_FILTER_RULES\").value" "$manifest") diff --git a/scripts/help.sh b/scripts/help.sh index 9d78629b0..adbea1c62 100644 --- a/scripts/help.sh +++ b/scripts/help.sh @@ -84,6 +84,12 @@ function flows_examples { function packets_examples { echo " Capture packets on port 8080:" echo " netobserv packets --port=8080" + echo " Capture HTTPS with OpenSSL plaintext via libssl uprobes:" + echo " netobserv packets --port=8443 --enable_openssl --peer_ip= --privileged --background" + echo " # test workload: examples/openssl-test-pod/" + echo " # readable output: output/plaintext/.jsonl (PlaintextDisplay field)" + echo " Capture with Wireshark key log embedded in pcapng:" + echo " netobserv packets --port=443 --tls-keylog=/path/to/keylog.txt" echo " Capture packets on specific nodes (labeled with 'netobserv=true') and port, for a maximum of 100MB:" echo " netobserv packets --node-selector=netobserv:true --port=80 --max-bytes=100000000" } @@ -129,6 +135,13 @@ function flowsAndPackets_collector_usage { echo " --max-bytes: maximum capture bytes (default: 50000000 = 50MB)" } +function packets_tls_usage { + echo " --enable_openssl: capture TLS plaintext via OpenSSL uprobes (default: false, requires --privileged; recommended: --peer_ip --port)" + echo " --tls_plaintext_min_bytes: drop TLS plaintext events shorter than N bytes (default: 0, agent env TLS_PLAINTEXT_MIN_BYTES)" + echo " --tls_plaintext_preview_bytes: PlaintextPreview length (0 = full captured payload) (default: 256)" + echo " --tls-keylog: path to SSLKEYLOGFILE for pcapng decryption (collector flag)" +} + # fmetrics collector options function metrics_collector_usage { echo " --background: run in background (default: false)" @@ -153,10 +166,10 @@ function filters_usage { echo " --icmp_code: filter ICMP code (default: n/a)" echo " --icmp_type: filter ICMP type (default: n/a)" echo " --node-selector: capture on specific nodes (default: n/a)" - echo " --peer_ip: filter peer IP (default: n/a)" + echo " --peer_ip: scope which processes get TLS hooks; filter peer IP (default: n/a; recommended with TLS flags)" echo " --peer_cidr: filter peer CIDR (default: n/a)" echo " --port_range: filter port range (default: n/a)" - echo " --port: filter port (default: n/a)" + echo " --port: filter exported plaintext and wire capture (default: n/a; recommended with TLS flags)" echo " --ports: filter on either of two ports (default: n/a)" echo " --protocol: filter protocol (default: n/a)" echo " --query: filter flows using a custom query (default: n/a)" @@ -211,6 +224,7 @@ function packets_usage { echo echo "Options:" flowsAndPackets_collector_usage + packets_tls_usage script_usage echo echo "Examples:"