feat(fluentbit): add retainMetadataInForwardMode to forward output pl…

apis/fluentbit/v1alpha2/clusteroutput_types_test.go

@@ -523,3 +523,52 @@ func TestLokiOutputWithStructuredMetadata_LoadAsYaml(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 	g.Expect(result).To(Equal(expected))
 }
+
+func TestForwardOutput_RetainMetadataInForwardMode(t *testing.T) {
+	g := NewGomegaWithT(t)
+	sl := plugins.NewSecretLoader(nil, "testnamespace")
+
+	// nil: field should be omitted
+	nilOut := ClusterOutputList{Items: []ClusterOutput{{
+		ObjectMeta: metav1.ObjectMeta{Name: "fwd-nil"},
+		Spec: OutputSpec{
+			Match:   "kube.*",
+			Forward: &output.Forward{Host: "fluentd.svc", Port: utils.ToPtr[int32](24224)},
+		},
+	}}}
+	resultNil, err := nilOut.Load(sl)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(resultNil).NotTo(ContainSubstring("Retain_Metadata_In_Forward_Mode"))
+
+	// false: field should be rendered as false
+	falseOut := ClusterOutputList{Items: []ClusterOutput{{
+		ObjectMeta: metav1.ObjectMeta{Name: "fwd-false"},
+		Spec: OutputSpec{
+			Match: "kube.*",
+			Forward: &output.Forward{
+				Host:                        "fluentd.svc",
+				Port:                        utils.ToPtr[int32](24224),
+				RetainMetadataInForwardMode: utils.ToPtr(false),
+			},
+		},
+	}}}
+	resultFalse, err := falseOut.Load(sl)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(resultFalse).To(ContainSubstring("Retain_Metadata_In_Forward_Mode    false"))
+
+	// true: field should be rendered as true
+	trueOut := ClusterOutputList{Items: []ClusterOutput{{
+		ObjectMeta: metav1.ObjectMeta{Name: "fwd-true"},
+		Spec: OutputSpec{
+			Match: "kube.*",
+			Forward: &output.Forward{
+				Host:                        "fluentd.svc",
+				Port:                        utils.ToPtr[int32](24224),
+				RetainMetadataInForwardMode: utils.ToPtr(true),
+			},
+		},
+	}}}
+	resultTrue, err := trueOut.Load(sl)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(resultTrue).To(ContainSubstring("Retain_Metadata_In_Forward_Mode    true"))
+}

apis/fluentbit/v1alpha2/plugins/output/forward_types.go

@@ -38,7 +38,9 @@ type Forward struct {
 	Password *plugins.Secret `json:"password,omitempty"`
 	// Default value of the auto-generated certificate common name (CN).
 	SelfHostname string `json:"selfHostname,omitempty"`
-	*plugins.TLS `json:"tls,omitempty"`
+	// Retain metadata when forwarding in forward mode.
+	RetainMetadataInForwardMode *bool `json:"retainMetadataInForwardMode,omitempty"`
+	*plugins.TLS                `json:"tls,omitempty"`
 	// Include fluentbit networking options for this output-plugin
 	*plugins.Networking `json:"networking,omitempty"`
 }
@@ -82,6 +84,7 @@ func (f *Forward) Params(sl plugins.SecretLoader) (*params.KVs, error) {
 	plugins.InsertKVField(kvs, "Time_as_Integer", f.TimeAsInteger)
 	plugins.InsertKVField(kvs, "Send_options", f.SendOptions)
 	plugins.InsertKVField(kvs, "Require_ack_response", f.RequireAckResponse)
+	plugins.InsertKVField(kvs, "Retain_Metadata_In_Forward_Mode", f.RetainMetadataInForwardMode)
 
 	return kvs, nil
 }

charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusteroutputs.yaml

@@ -1156,6 +1156,9 @@ spec:
                       Enables at-least-once and receiving server can control rate of traffic.
                       (Requires Fluentd v0.14.0+ server)
                     type: boolean
+                  retainMetadataInForwardMode:
+                    description: Retain metadata when forwarding in forward mode.
+                    type: boolean
                   selfHostname:
                     description: Default value of the auto-generated certificate common
                       name (CN).

charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_outputs.yaml

@@ -1156,6 +1156,9 @@ spec:
                       Enables at-least-once and receiving server can control rate of traffic.
                       (Requires Fluentd v0.14.0+ server)
                     type: boolean
+                  retainMetadataInForwardMode:
+                    description: Retain metadata when forwarding in forward mode.
+                    type: boolean
                   selfHostname:
                     description: Default value of the auto-generated certificate common
                       name (CN).

charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml

@@ -1154,6 +1154,9 @@ spec:
                       Enables at-least-once and receiving server can control rate of traffic.
                       (Requires Fluentd v0.14.0+ server)
                     type: boolean
+                  retainMetadataInForwardMode:
+                    description: Retain metadata when forwarding in forward mode.
+                    type: boolean
                   selfHostname:
                     description: Default value of the auto-generated certificate common
                       name (CN).

charts/fluent-operator/crds/fluentbit.fluent.io_outputs.yaml

@@ -1154,6 +1154,9 @@ spec:
                       Enables at-least-once and receiving server can control rate of traffic.
                       (Requires Fluentd v0.14.0+ server)
                     type: boolean
+                  retainMetadataInForwardMode:
+                    description: Retain metadata when forwarding in forward mode.
+                    type: boolean
                   selfHostname:
                     description: Default value of the auto-generated certificate common
                       name (CN).

charts/fluent-operator/templates/fluentbit-output-forward.yaml

@@ -13,6 +13,9 @@ spec:
   forward:
     host: {{ printf "%s.%s.svc.cluster.local" .Values.fluentd.name .Release.Namespace | quote }}
     port: {{ .Values.fluentd.forward.port }}
+    {{- if not (kindIs "invalid" .Values.fluentd.forward.retainMetadataInForwardMode) }}
+    retainMetadataInForwardMode: {{ .Values.fluentd.forward.retainMetadataInForwardMode }}
+    {{- end }}
 {{- end }}
 {{- end }}
 {{- end }}

charts/fluent-operator/values.yaml

@@ -627,6 +627,9 @@ fluentd:
   forward:
     # -- Forward input port
     port: 24224
+    # -- Retain metadata when forwarding in forward mode.
+    # Set to false for Fluentd compatibility with Fluent Bit v5.0.5+.
+    retainMetadataInForwardMode: ~
   # -- Namespaces to watch for Fluentd resources
   watchedNamespaces:
     - kube-system

config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml

@@ -1155,6 +1155,9 @@ spec:
                       Enables at-least-once and receiving server can control rate of traffic.
                       (Requires Fluentd v0.14.0+ server)
                     type: boolean
+                  retainMetadataInForwardMode:
+                    description: Retain metadata when forwarding in forward mode.
+                    type: boolean
                   selfHostname:
                     description: Default value of the auto-generated certificate common
                       name (CN).

config/crd/bases/fluentbit.fluent.io_outputs.yaml

@@ -1155,6 +1155,9 @@ spec:
                       Enables at-least-once and receiving server can control rate of traffic.
                       (Requires Fluentd v0.14.0+ server)
                     type: boolean
+                  retainMetadataInForwardMode:
+                    description: Retain metadata when forwarding in forward mode.
+                    type: boolean
                   selfHostname:
                     description: Default value of the auto-generated certificate common
                       name (CN).

docs/plugins/fluentbit/output/forward.md

@@ -16,5 +16,6 @@ Forward is the protocol used by Fluentd to route messages between peers. <br />
 | username | Specify the username to present to a Fluentd server that enables user_auth. | *[plugins.Secret](../secret.md) |
 | password | Specify the password corresponding to the username. | *[plugins.Secret](../secret.md) |
 | selfHostname | Default value of the auto-generated certificate common name (CN). | string |
+| retainMetadataInForwardMode | Retain metadata when forwarding in forward mode. | *bool |
 | tls |  | *[plugins.TLS](../tls.md) |
 | networking | Include fluentbit networking options for this output-plugin | *plugins.Networking |

manifests/setup/setup.yaml

@@ -5355,6 +5355,9 @@ spec:
                       Enables at-least-once and receiving server can control rate of traffic.
                       (Requires Fluentd v0.14.0+ server)
                     type: boolean
+                  retainMetadataInForwardMode:
+                    description: Retain metadata when forwarding in forward mode.
+                    type: boolean
                   selfHostname:
                     description: Default value of the auto-generated certificate common
                       name (CN).
@@ -35442,6 +35445,9 @@ spec:
                       Enables at-least-once and receiving server can control rate of traffic.
                       (Requires Fluentd v0.14.0+ server)
                     type: boolean
+                  retainMetadataInForwardMode:
+                    description: Retain metadata when forwarding in forward mode.
+                    type: boolean
                   selfHostname:
                     description: Default value of the auto-generated certificate common
                       name (CN).