From 49d56fddbe48e6234005e9ad8b9459dc0dc4d258 Mon Sep 17 00:00:00 2001 From: Ryan Lymburner Date: Tue, 27 May 2025 13:52:15 -0700 Subject: [PATCH 1/8] Initial changes and unit tests --- ...ion-networking.k8s.aws_serviceexports.yaml | 31 +++ docs/api-types/service-export.md | 51 ++++- files/examples/inventory-ver2-export.yaml | 4 + .../multi-protocol-service-export.yaml | 14 ++ files/examples/service-1-export.yaml | 4 + files/examples/service-2-export.yaml | 4 + files/examples/tls-rate2-export.yaml | 6 +- ...ion-networking.k8s.aws_serviceexports.yaml | 31 +++ .../v1alpha1/serviceexport_types.go | 22 ++ pkg/gateway/model_build_targetgroup.go | 130 +++++++++++ pkg/gateway/model_build_targetgroup_test.go | 213 ++++++++++++++++++ 11 files changed, 500 insertions(+), 10 deletions(-) create mode 100644 files/examples/multi-protocol-service-export.yaml diff --git a/config/crds/bases/application-networking.k8s.aws_serviceexports.yaml b/config/crds/bases/application-networking.k8s.aws_serviceexports.yaml index b3b01dab..8871b49c 100644 --- a/config/crds/bases/application-networking.k8s.aws_serviceexports.yaml +++ b/config/crds/bases/application-networking.k8s.aws_serviceexports.yaml @@ -38,6 +38,37 @@ spec: type: string metadata: type: object + spec: + description: spec defines the desired state of ServiceExport + properties: + exportedPorts: + description: |- + exportedPorts defines which ports of the service should be exported and what route types they should be used with. + If not specified, the controller will use the port from the annotation "application-networking.k8s.aws/port" + and create HTTP target groups for backward compatibility. + items: + description: ExportedPort defines a port to be exported and the + route type it should be used with + properties: + port: + description: port is the port number to export + format: int32 + type: integer + routeType: + description: |- + routeType is the type of route this port should be used with + Valid values are "HTTP", "GRPC", "TLS" + enum: + - HTTP + - GRPC + - TLS + type: string + required: + - port + - routeType + type: object + type: array + type: object status: description: |- status describes the current state of an exported service. diff --git a/docs/api-types/service-export.md b/docs/api-types/service-export.md index e57ce0eb..89046a4c 100644 --- a/docs/api-types/service-export.md +++ b/docs/api-types/service-export.md @@ -12,21 +12,34 @@ for example, using target groups in the VPC Lattice setup outside Kubernetes. Note that ServiceExport is not the implementation of Kubernetes [Multicluster Service APIs](https://multicluster.sigs.k8s.io/concepts/multicluster-services-api/); instead AWS Gateway API Controller uses its own version of the resource for the purpose of Gateway API integration. - -### Limitations -* The exported Service can only be used in HTTPRoutes. GRPCRoute is currently not supported. -* Limited to one ServiceExport per Service. If you need multiple exports representing each port, - you should create multiple Service-ServiceExport pairs. - -### Annotations +### Annotations (Legacy Method) * `application-networking.k8s.aws/port` Represents which port of the exported Service will be used. When a comma-separated list of ports is provided, the traffic will be distributed to all ports in the list. + + **Note:** This annotation is supported for backward compatibility. For new deployments, it's recommended to use the `spec.exportedPorts` field instead. + +## Spec Fields + +### exportedPorts + +The `exportedPorts` field allows you to explicitly define which ports of the service should be exported and what route types they should be used with. This is useful when you have a service with multiple ports serving different protocols. -## Example Configuration +Each exported port has the following fields: +* `port`: The port number to export +* `routeType`: The type of route this port should be used with. Valid values are: + * `HTTP`: For HTTP traffic + * `GRPC`: For gRPC traffic + * `TLS`: For TLS traffic -The following yaml will create a ServiceExport for a Service named `service-1`: +If `exportedPorts` is not specified, the controller will use the port from the annotation "application-networking.k8s.aws/port" and create HTTP target groups for backward compatibility. + +## Example Configurations + +### Legacy Configuration (Using Annotations) + +The following yaml will create a ServiceExport for a Service named `service-1` using the legacy annotation method: ```yaml apiVersion: application-networking.k8s.aws/v1alpha1 kind: ServiceExport @@ -36,3 +49,23 @@ metadata: application-networking.k8s.aws/port: "9200" spec: {} ``` + +### Using exportedPorts + +The following yaml will create a ServiceExport for a Service named `service-1` with multiple ports for different route types: +```yaml +apiVersion: application-networking.k8s.aws/v1alpha1 +kind: ServiceExport +metadata: + name: service-1 +spec: + exportedPorts: + - port: 80 + routeType: HTTP + - port: 8081 + routeType: GRPC +``` + +This configuration will: +1. Export port 80 to be used with HTTP routes +2. Export port 8081 to be used with gRPC routes diff --git a/files/examples/inventory-ver2-export.yaml b/files/examples/inventory-ver2-export.yaml index 443e01c3..a0a822d6 100644 --- a/files/examples/inventory-ver2-export.yaml +++ b/files/examples/inventory-ver2-export.yaml @@ -4,3 +4,7 @@ metadata: name: inventory-ver2 annotations: application-networking.k8s.aws/federation: "amazon-vpc-lattice" +spec: + exportedPorts: + - port: 80 + routeType: HTTP diff --git a/files/examples/multi-protocol-service-export.yaml b/files/examples/multi-protocol-service-export.yaml new file mode 100644 index 00000000..132e9551 --- /dev/null +++ b/files/examples/multi-protocol-service-export.yaml @@ -0,0 +1,14 @@ +apiVersion: application-networking.k8s.aws/v1alpha1 +kind: ServiceExport +metadata: + name: multi-protocol-service + annotations: + application-networking.k8s.aws/federation: "amazon-vpc-lattice" +spec: + exportedPorts: + - port: 80 + routeType: HTTP + - port: 8081 + routeType: GRPC + - port: 443 + routeType: TLS diff --git a/files/examples/service-1-export.yaml b/files/examples/service-1-export.yaml index 8732c379..69b66117 100644 --- a/files/examples/service-1-export.yaml +++ b/files/examples/service-1-export.yaml @@ -4,3 +4,7 @@ metadata: name: service-1 annotations: application-networking.k8s.aws/federation: "amazon-vpc-lattice" +spec: + exportedPorts: + - port: 80 + routeType: HTTP diff --git a/files/examples/service-2-export.yaml b/files/examples/service-2-export.yaml index a1824db6..91a4084b 100644 --- a/files/examples/service-2-export.yaml +++ b/files/examples/service-2-export.yaml @@ -4,3 +4,7 @@ metadata: name: service-2 annotations: application-networking.k8s.aws/federation: "amazon-vpc-lattice" +spec: + exportedPorts: + - port: 80 + routeType: HTTP diff --git a/files/examples/tls-rate2-export.yaml b/files/examples/tls-rate2-export.yaml index 352944fc..373c322e 100644 --- a/files/examples/tls-rate2-export.yaml +++ b/files/examples/tls-rate2-export.yaml @@ -3,4 +3,8 @@ kind: ServiceExport metadata: name: tls-rate2 annotations: - application-networking.k8s.aws/federation: "amazon-vpc-lattice" \ No newline at end of file + application-networking.k8s.aws/federation: "amazon-vpc-lattice" +spec: + exportedPorts: + - port: 443 + routeType: TLS diff --git a/helm/crds/application-networking.k8s.aws_serviceexports.yaml b/helm/crds/application-networking.k8s.aws_serviceexports.yaml index b3b01dab..8871b49c 100644 --- a/helm/crds/application-networking.k8s.aws_serviceexports.yaml +++ b/helm/crds/application-networking.k8s.aws_serviceexports.yaml @@ -38,6 +38,37 @@ spec: type: string metadata: type: object + spec: + description: spec defines the desired state of ServiceExport + properties: + exportedPorts: + description: |- + exportedPorts defines which ports of the service should be exported and what route types they should be used with. + If not specified, the controller will use the port from the annotation "application-networking.k8s.aws/port" + and create HTTP target groups for backward compatibility. + items: + description: ExportedPort defines a port to be exported and the + route type it should be used with + properties: + port: + description: port is the port number to export + format: int32 + type: integer + routeType: + description: |- + routeType is the type of route this port should be used with + Valid values are "HTTP", "GRPC", "TLS" + enum: + - HTTP + - GRPC + - TLS + type: string + required: + - port + - routeType + type: object + type: array + type: object status: description: |- status describes the current state of an exported service. diff --git a/pkg/apis/applicationnetworking/v1alpha1/serviceexport_types.go b/pkg/apis/applicationnetworking/v1alpha1/serviceexport_types.go index 9492c2da..09b2a99e 100644 --- a/pkg/apis/applicationnetworking/v1alpha1/serviceexport_types.go +++ b/pkg/apis/applicationnetworking/v1alpha1/serviceexport_types.go @@ -30,6 +30,9 @@ type ServiceExport struct { apimachineryv1.TypeMeta `json:",inline"` // +optional apimachineryv1.ObjectMeta `json:"metadata,omitempty"` + // spec defines the desired state of ServiceExport + // +optional + Spec ServiceExportSpec `json:"spec,omitempty"` // status describes the current state of an exported service. // Service configuration comes from the Service that had the same // name and namespace as this ServiceExport. @@ -38,6 +41,25 @@ type ServiceExport struct { Status ServiceExportStatus `json:"status,omitempty"` } +// ServiceExportSpec defines the desired state of ServiceExport +type ServiceExportSpec struct { + // exportedPorts defines which ports of the service should be exported and what route types they should be used with. + // If not specified, the controller will use the port from the annotation "application-networking.k8s.aws/port" + // and create HTTP target groups for backward compatibility. + // +optional + ExportedPorts []ExportedPort `json:"exportedPorts,omitempty"` +} + +// ExportedPort defines a port to be exported and the route type it should be used with +type ExportedPort struct { + // port is the port number to export + Port int32 `json:"port"` + // routeType is the type of route this port should be used with + // Valid values are "HTTP", "GRPC", "TLS" + // +kubebuilder:validation:Enum=HTTP;GRPC;TLS + RouteType string `json:"routeType"` +} + // ServiceExportStatus contains the current status of an export. type ServiceExportStatus struct { // +optional diff --git a/pkg/gateway/model_build_targetgroup.go b/pkg/gateway/model_build_targetgroup.go index ef885961..99b1e1b9 100644 --- a/pkg/gateway/model_build_targetgroup.go +++ b/pkg/gateway/model_build_targetgroup.go @@ -98,10 +98,40 @@ func (b *SvcExportTargetGroupBuilder) BuildTargetGroup(ctx context.Context, svcE tgp: policy.NewTargetGroupPolicyHandler(b.log, b.client), } + // If exportedPorts is defined, we need to handle it differently + // For now, we'll just return the first target group for backward compatibility + // This is used for reconciliation of existing target groups + if len(svcExport.Spec.ExportedPorts) > 0 { + return task.buildTargetGroupForExportedPort(ctx, svcExport.Spec.ExportedPorts[0]) + } + return task.buildTargetGroup(ctx) } func (t *svcExportTargetGroupModelBuildTask) run(ctx context.Context) error { + // Check if we have exportedPorts defined in the spec + if len(t.serviceExport.Spec.ExportedPorts) > 0 { + // Create target groups for each exported port + for _, exportedPort := range t.serviceExport.Spec.ExportedPorts { + tg, err := t.buildTargetGroupForExportedPort(ctx, exportedPort) + if err != nil { + return fmt.Errorf("failed to build target group for service export %s-%s port %d due to %w", + t.serviceExport.Name, t.serviceExport.Namespace, exportedPort.Port, err) + } + + if !tg.IsDeleted { + err = t.buildTargetsForPort(ctx, tg.ID(), exportedPort.Port) + if err != nil { + t.log.Debugf(ctx, "Failed to build targets for service export %s-%s port %d due to %s", + t.serviceExport.Name, t.serviceExport.Namespace, exportedPort.Port, err) + return err + } + } + } + return nil + } + + // Fall back to legacy behavior if no exportedPorts are defined tg, err := t.buildTargetGroup(ctx) if err != nil { return fmt.Errorf("failed to build target group for service export %s-%s due to %w", @@ -129,6 +159,106 @@ func (t *svcExportTargetGroupModelBuildTask) buildTargets(ctx context.Context, s return nil } +func (t *svcExportTargetGroupModelBuildTask) buildTargetGroupForExportedPort(ctx context.Context, exportedPort anv1alpha1.ExportedPort) (*model.TargetGroup, error) { + svc := &corev1.Service{} + noSvcFoundAndDeleting := false + if err := t.client.Get(ctx, k8s.NamespacedName(t.serviceExport), svc); err != nil { + if apierrors.IsNotFound(err) && !t.serviceExport.DeletionTimestamp.IsZero() { + // if we're deleting, it's OK if the service isn't there + noSvcFoundAndDeleting = true + } else { // either it's some other error or we aren't deleting + return nil, fmt.Errorf("failed to find corresponding k8sService %s, error :%w ", + k8s.NamespacedName(t.serviceExport), err) + } + } + + var ipAddressType string + var err error + if noSvcFoundAndDeleting { + ipAddressType = "IPV4" // just pick a default + } else { + ipAddressType, err = buildTargetGroupIpAddressType(svc) + if err != nil { + return nil, err + } + } + + tgp, err := t.tgp.ObjResolvedPolicy(ctx, t.serviceExport) + if err != nil { + return nil, err + } + + // Get health check config from policy + _, _, healthCheckConfig, err := parseTargetGroupConfig(tgp) + if err != nil { + return nil, err + } + + // Set protocol and protocolVersion based on routeType + var protocol, protocolVersion string + switch exportedPort.RouteType { + case "HTTP": + protocol = vpclattice.TargetGroupProtocolHttp + protocolVersion = vpclattice.TargetGroupProtocolVersionHttp1 + case "GRPC": + protocol = vpclattice.TargetGroupProtocolHttp + protocolVersion = vpclattice.TargetGroupProtocolVersionGrpc + case "TLS": + protocol = vpclattice.TargetGroupProtocolTcp + protocolVersion = "" + default: + return nil, fmt.Errorf("unsupported route type: %s", exportedPort.RouteType) + } + + spec := model.TargetGroupSpec{ + Type: model.TargetGroupTypeIP, + Port: exportedPort.Port, + Protocol: protocol, + ProtocolVersion: protocolVersion, + IpAddressType: ipAddressType, + HealthCheckConfig: healthCheckConfig, + } + spec.VpcId = config.VpcID + spec.K8SSourceType = model.SourceTypeSvcExport + spec.K8SClusterName = config.ClusterName + spec.K8SServiceName = t.serviceExport.Name + spec.K8SServiceNamespace = t.serviceExport.Namespace + spec.K8SProtocolVersion = protocolVersion + + // Add a tag for the route type to help with identification + // This is not used by the controller but can be helpful for debugging + if exportedPort.RouteType != "" { + spec.K8SProtocolVersion = exportedPort.RouteType + } + + stackTG, err := model.NewTargetGroup(t.stack, spec) + if err != nil { + return nil, err + } + + stackTG.IsDeleted = !t.serviceExport.DeletionTimestamp.IsZero() + return stackTG, nil +} + +func (t *svcExportTargetGroupModelBuildTask) buildTargetsForPort(ctx context.Context, stackTgId string, port int32) error { + // This is similar to buildTargets but filters endpoints by the specified port + targetsBuilder := NewTargetsBuilder(t.log, t.client, t.stack) + + // We need to create a modified ServiceExport with the port annotation set to the specific port + // This is a bit of a hack, but it allows us to reuse the existing BuildForServiceExport method + modifiedServiceExport := t.serviceExport.DeepCopy() + if modifiedServiceExport.Annotations == nil { + modifiedServiceExport.Annotations = make(map[string]string) + } + modifiedServiceExport.Annotations[portAnnotationsKey] = fmt.Sprintf("%d", port) + + _, err := targetsBuilder.BuildForServiceExport(ctx, modifiedServiceExport, stackTgId) + if err != nil { + return err + } + return nil +} + func (t *svcExportTargetGroupModelBuildTask) buildTargetGroup(ctx context.Context) (*model.TargetGroup, error) { svc := &corev1.Service{} noSvcFoundAndDeleting := false diff --git a/pkg/gateway/model_build_targetgroup_test.go b/pkg/gateway/model_build_targetgroup_test.go index a5ab93f8..ff877c21 100644 --- a/pkg/gateway/model_build_targetgroup_test.go +++ b/pkg/gateway/model_build_targetgroup_test.go @@ -628,6 +628,219 @@ func Test_ServiceImportToTGBuildReturnsError(t *testing.T) { } } +func Test_TGModelByServiceExportWithExportedPorts(t *testing.T) { + config.VpcID = "vpc-id" + config.ClusterName = "cluster-name" + + tests := []struct { + name string + svcExport *anv1alpha1.ServiceExport + svc *corev1.Service + wantErrIsNil bool + wantTargetGroupCount int + wantPorts []int32 + wantRouteTypes []string + }{ + { + name: "ServiceExport with multiple exportedPorts", + svcExport: &anv1alpha1.ServiceExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multi-protocol", + Namespace: "ns1", + }, + Spec: anv1alpha1.ServiceExportSpec{ + ExportedPorts: []anv1alpha1.ExportedPort{ + { + Port: 80, + RouteType: "HTTP", + }, + { + Port: 8081, + RouteType: "GRPC", + }, + { + Port: 443, + RouteType: "TLS", + }, + }, + }, + }, + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multi-protocol", + Namespace: "ns1", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + { + Name: "grpc", + Port: 8081, + }, + { + Name: "tls", + Port: 443, + }, + }, + IPFamilies: []corev1.IPFamily{ + corev1.IPv4Protocol, + }, + }, + }, + wantErrIsNil: true, + wantTargetGroupCount: 3, + wantPorts: []int32{80, 8081, 443}, + wantRouteTypes: []string{"HTTP", "GRPC", "TLS"}, + }, + { + name: "ServiceExport with single exportedPort", + svcExport: &anv1alpha1.ServiceExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "single-port", + Namespace: "ns1", + }, + Spec: anv1alpha1.ServiceExportSpec{ + ExportedPorts: []anv1alpha1.ExportedPort{ + { + Port: 80, + RouteType: "HTTP", + }, + }, + }, + }, + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "single-port", + Namespace: "ns1", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + IPFamilies: []corev1.IPFamily{ + corev1.IPv4Protocol, + }, + }, + }, + wantErrIsNil: true, + wantTargetGroupCount: 1, + wantPorts: []int32{80}, + wantRouteTypes: []string{"HTTP"}, + }, + { + name: "ServiceExport with no exportedPorts (legacy behavior)", + svcExport: &anv1alpha1.ServiceExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "legacy", + Namespace: "ns1", + Annotations: map[string]string{ + "application-networking.k8s.aws/port": "80", + }, + }, + }, + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "legacy", + Namespace: "ns1", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + IPFamilies: []corev1.IPFamily{ + corev1.IPv4Protocol, + }, + }, + }, + wantErrIsNil: true, + wantTargetGroupCount: 1, + wantPorts: []int32{80}, + wantRouteTypes: []string{vpclattice.TargetGroupProtocolVersionHttp1}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + + k8sSchema := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sSchema) + anv1alpha1.Install(k8sSchema) + k8sClient := testclient.NewClientBuilder().WithScheme(k8sSchema).Build() + + if tt.svc != nil { + assert.NoError(t, k8sClient.Create(ctx, tt.svc.DeepCopy())) + } + + builder := NewSvcExportTargetGroupBuilder(gwlog.FallbackLogger, k8sClient) + + stack, err := builder.Build(ctx, tt.svcExport) + if !tt.wantErrIsNil { + assert.NotNil(t, err) + return + } + assert.Nil(t, err) + + var targetGroups []*model.TargetGroup + err = stack.ListResources(&targetGroups) + assert.Nil(t, err) + assert.Equal(t, tt.wantTargetGroupCount, len(targetGroups)) + + // Create maps to track which ports and route types we've seen + seenPorts := make(map[int32]bool) + seenRouteTypes := make(map[string]bool) + + for _, tg := range targetGroups { + // Check common properties + assert.Equal(t, model.TargetGroupTypeIP, tg.Spec.Type) + assert.Equal(t, model.SourceTypeSvcExport, tg.Spec.K8SSourceType) + assert.Equal(t, config.ClusterName, tg.Spec.K8SClusterName) + assert.Equal(t, tt.svcExport.Name, tg.Spec.K8SServiceName) + assert.Equal(t, tt.svcExport.Namespace, tg.Spec.K8SServiceNamespace) + assert.False(t, tg.IsDeleted) + + // Track the port and route type + seenPorts[tg.Spec.Port] = true + seenRouteTypes[tg.Spec.K8SProtocolVersion] = true + + // Check protocol and protocolVersion based on route type + switch tg.Spec.K8SProtocolVersion { + case "HTTP": + assert.Equal(t, vpclattice.TargetGroupProtocolHttp, tg.Spec.Protocol) + assert.Equal(t, vpclattice.TargetGroupProtocolVersionHttp1, tg.Spec.ProtocolVersion) + case "GRPC": + assert.Equal(t, vpclattice.TargetGroupProtocolHttp, tg.Spec.Protocol) + assert.Equal(t, vpclattice.TargetGroupProtocolVersionGrpc, tg.Spec.ProtocolVersion) + case "TLS": + assert.Equal(t, vpclattice.TargetGroupProtocolTcp, tg.Spec.Protocol) + assert.Equal(t, "", tg.Spec.ProtocolVersion) + case vpclattice.TargetGroupProtocolVersionHttp1: + // Legacy behavior + assert.Equal(t, vpclattice.TargetGroupProtocolHttp, tg.Spec.Protocol) + } + } + + // Verify that we've seen all expected ports and route types + for _, port := range tt.wantPorts { + assert.True(t, seenPorts[port], "Expected port %d not found", port) + } + + for _, routeType := range tt.wantRouteTypes { + assert.True(t, seenRouteTypes[routeType], "Expected route type %s not found", routeType) + } + }) + } +} + func Test_buildTargetGroupIpAddressType(t *testing.T) { type args struct { svc *corev1.Service From 710dffa7d011a9c08cab43d185432116ce1ff6f8 Mon Sep 17 00:00:00 2001 From: Ryan Lymburner Date: Tue, 27 May 2025 15:28:25 -0700 Subject: [PATCH 2/8] Added GRPC ServiceExport integration test --- .../grpcroute_serviceexport_test.go | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 test/suites/integration/grpcroute_serviceexport_test.go diff --git a/test/suites/integration/grpcroute_serviceexport_test.go b/test/suites/integration/grpcroute_serviceexport_test.go new file mode 100644 index 00000000..ddaef177 --- /dev/null +++ b/test/suites/integration/grpcroute_serviceexport_test.go @@ -0,0 +1,147 @@ +package integration + +import ( + "fmt" + "log" + "os" + "time" + + "github.com/aws/aws-application-networking-k8s/pkg/model/core" + "github.com/aws/aws-application-networking-k8s/test/pkg/test" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/vpclattice" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + + anv1alpha1 "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1" +) + +var _ = Describe("GRPCRoute Service Export/Import Test", Ordered, func() { + var ( + grpcDeployment *appsv1.Deployment + grpcSvc *v1.Service + grpcRoute *gwv1.GRPCRoute + serviceExport *anv1alpha1.ServiceExport + serviceImport *anv1alpha1.ServiceImport + ) + + It("Create k8s resource", func() { + // Create a gRPC service and deployment + grpcDeployment, grpcSvc = testFramework.NewGrpcHelloWorld(test.GrpcAppOptions{AppName: "my-grpc-exportedports", Namespace: k8snamespace}) + testFramework.ExpectCreated(ctx, grpcDeployment, grpcSvc) + + // Create ServiceImport + serviceImport = testFramework.CreateServiceImport(grpcSvc) + testFramework.ExpectCreated(ctx, serviceImport) + + // Create ServiceExport with exportedPorts field + serviceExport = &anv1alpha1.ServiceExport{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "application-networking.k8s.aws/v1alpha1", + Kind: "ServiceExport", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: grpcSvc.Name, + Namespace: grpcSvc.Namespace, + Annotations: map[string]string{ + "application-networking.k8s.aws/federation": "amazon-vpc-lattice", + }, + }, + Spec: anv1alpha1.ServiceExportSpec{ + ExportedPorts: []anv1alpha1.ExportedPort{ + { + Port: grpcSvc.Spec.Ports[0].Port, + RouteType: "GRPC", + }, + }, + }, + } + testFramework.ExpectCreated(ctx, serviceExport) + + // Create GRPCRoute + grpcRoute = testFramework.NewGRPCRoute(k8snamespace, testGateway, []gwv1.GRPCRouteRule{ + { + Matches: []gwv1.GRPCRouteMatch{ + { + Method: &gwv1.GRPCMethodMatch{ + Service: lo.ToPtr("helloworld.Greeter"), + Method: lo.ToPtr("SayHello"), + Type: lo.ToPtr(gwv1.GRPCMethodMatchExact), + }, + }, + }, + BackendRefs: []gwv1.GRPCBackendRef{ + { + BackendRef: gwv1.BackendRef{ + BackendObjectReference: gwv1.BackendObjectReference{ + Name: gwv1.ObjectName(grpcSvc.Name), + Namespace: lo.ToPtr(gwv1.Namespace(grpcSvc.Namespace)), + Kind: lo.ToPtr(gwv1.Kind("ServiceImport")), + Port: lo.ToPtr(gwv1.PortNumber(grpcSvc.Spec.Ports[0].Port)), + }, + }, + }, + }, + }, + }) + testFramework.ExpectCreated(ctx, grpcRoute) + }) + + It("Verify lattice resource & traffic", func() { + route, _ := core.NewRoute(grpcRoute) + vpcLatticeService := testFramework.GetVpcLatticeService(ctx, route) + fmt.Printf("vpcLatticeService: %v \n", vpcLatticeService) + + // Get the target group and verify it's configured for gRPC + tgSummary := testFramework.GetTargetGroupWithProtocol(ctx, grpcSvc, "http", "grpc") + tg, err := testFramework.LatticeClient.GetTargetGroup(&vpclattice.GetTargetGroupInput{ + TargetGroupIdentifier: aws.String(*tgSummary.Id), + }) + Expect(tg).To(Not(BeNil())) + Expect(err).To(BeNil()) + Expect(*tgSummary.VpcIdentifier).To(Equal(os.Getenv("CLUSTER_VPC_ID"))) + + // Verify the target group is configured for gRPC + Expect(*tgSummary.Protocol).To(Equal("HTTP")) + Expect(*tg.Config.ProtocolVersion).To(Equal("GRPC")) + + // Verify targets are registered + Eventually(func(g Gomega) { + targets := testFramework.GetTargets(ctx, tgSummary, grpcDeployment) + for _, target := range targets { + g.Expect(*target.Port).To(BeEquivalentTo(grpcSvc.Spec.Ports[0].TargetPort.IntVal)) + } + }).WithTimeout(3 * time.Minute).WithOffset(1).Should(Succeed()) + + log.Println("Verifying traffic") + grpcurlCmdOptions := test.RunGrpcurlCmdOptions{ + GrpcServerHostName: *vpcLatticeService.DnsEntry.DomainName, + GrpcServerPort: "443", + Service: "helloworld.Greeter", + Method: "SayHello", + ReqParamsJsonString: `{"name": "ExportedPorts"}`, + UseTLS: true, + } + Eventually(func(g Gomega) { + stdoutStr, stderrStr, err := testFramework.RunGrpcurlCmd(grpcurlCmdOptions) + g.Expect(err).To(BeNil()) + g.Expect(stderrStr).To(BeEmpty()) + g.Expect(stdoutStr).To(ContainSubstring("ExportedPorts")) + }).Should(Succeed()) + }) + + AfterAll(func() { + testFramework.ExpectDeletedThenNotFound(ctx, + grpcRoute, + grpcDeployment, + grpcSvc, + serviceImport, + serviceExport, + ) + }) +}) From d3143a2adb94f36e7840e9aa41625ecefe45c24c Mon Sep 17 00:00:00 2001 From: Ryan Lymburner Date: Wed, 28 May 2025 12:21:12 -0700 Subject: [PATCH 3/8] Increase suite timeout to 90 minutes --- pkg/controllers/suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/suite_test.go b/pkg/controllers/suite_test.go index 764ca33c..e37b4215 100644 --- a/pkg/controllers/suite_test.go +++ b/pkg/controllers/suite_test.go @@ -67,7 +67,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) -}, 60) +}, 90) var _ = AfterSuite(func() { By("tearing down the test environment") From 9d6013b04fde56922361614a8a540b53d6b5cbbb Mon Sep 17 00:00:00 2001 From: Ryan Lymburner Date: Wed, 28 May 2025 15:07:08 -0700 Subject: [PATCH 4/8] Increase suite timeout to 90 minutes --- Makefile | 1 + pkg/controllers/suite_test.go | 2 +- test/suites/integration/byoc_test.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3f905362..7e536fba 100644 --- a/Makefile +++ b/Makefile @@ -125,6 +125,7 @@ e2e-test: ## Run e2e tests against cluster pointed to by ~/.kube/config ./suites/integration/... \ --ginkgo.focus="${FOCUS}" \ --ginkgo.skip="${SKIP}" \ + --ginkgo.timeout=90m \ --ginkgo.v .SILENT: diff --git a/pkg/controllers/suite_test.go b/pkg/controllers/suite_test.go index e37b4215..775ebce6 100644 --- a/pkg/controllers/suite_test.go +++ b/pkg/controllers/suite_test.go @@ -67,7 +67,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) -}, 90) +}, 5400) var _ = AfterSuite(func() { By("tearing down the test environment") diff --git a/test/suites/integration/byoc_test.go b/test/suites/integration/byoc_test.go index 6f1aad91..977b1d50 100644 --- a/test/suites/integration/byoc_test.go +++ b/test/suites/integration/byoc_test.go @@ -80,7 +80,7 @@ var _ = Describe("Bring your own certificate (BYOC)", Ordered, func() { // get lattice service dns name for route53 cname svc := testFramework.GetVpcLatticeService(context.TODO(), core.NewHTTPRoute(gwv1.HTTPRoute(*httpRoute))) latticeSvcDns = *svc.DnsEntry.DomainName - log.Infof(ctx, "depoloyed lattice service, dns name: %s", latticeSvcDns) + log.Infof(ctx, "deployed lattice service, dns name: %s", latticeSvcDns) // create route 53 hosted zone and cname hz, err := createHostedZoneIfNotExists(r53Client, hostedZoneName) From 1f0a19bed6444369f7d57b0d02aa46475ead0682 Mon Sep 17 00:00:00 2001 From: Ryan Lymburner Date: Thu, 29 May 2025 10:34:27 -0700 Subject: [PATCH 5/8] Update comments --- pkg/gateway/model_build_targetgroup.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/gateway/model_build_targetgroup.go b/pkg/gateway/model_build_targetgroup.go index 99b1e1b9..37f0ddf9 100644 --- a/pkg/gateway/model_build_targetgroup.go +++ b/pkg/gateway/model_build_targetgroup.go @@ -164,9 +164,9 @@ func (t *svcExportTargetGroupModelBuildTask) buildTargetGroupForExportedPort(ctx noSvcFoundAndDeleting := false if err := t.client.Get(ctx, k8s.NamespacedName(t.serviceExport), svc); err != nil { if apierrors.IsNotFound(err) && !t.serviceExport.DeletionTimestamp.IsZero() { - // if we're deleting, it's OK if the service isn't there + // If we're deleting, it's OK if the service isn't there noSvcFoundAndDeleting = true - } else { // either it's some other error or we aren't deleting + } else { // Either it's some other error or we aren't deleting return nil, fmt.Errorf("failed to find corresponding k8sService %s, error :%w ", k8s.NamespacedName(t.serviceExport), err) } @@ -245,7 +245,7 @@ func (t *svcExportTargetGroupModelBuildTask) buildTargetsForPort(ctx context.Con targetsBuilder := NewTargetsBuilder(t.log, t.client, t.stack) // We need to create a modified ServiceExport with the port annotation set to the specific port - // This is a bit of a hack, but it allows us to reuse the existing BuildForServiceExport method + // This allows us to reuse the existing BuildForServiceExport method modifiedServiceExport := t.serviceExport.DeepCopy() if modifiedServiceExport.Annotations == nil { modifiedServiceExport.Annotations = make(map[string]string) @@ -264,9 +264,9 @@ func (t *svcExportTargetGroupModelBuildTask) buildTargetGroup(ctx context.Contex noSvcFoundAndDeleting := false if err := t.client.Get(ctx, k8s.NamespacedName(t.serviceExport), svc); err != nil { if apierrors.IsNotFound(err) && !t.serviceExport.DeletionTimestamp.IsZero() { - // if we're deleting, it's OK if the service isn't there + // If we're deleting, it's OK if the service isn't there noSvcFoundAndDeleting = true - } else { // either it's some other error or we aren't deleting + } else { // Either it's some other error or we aren't deleting return nil, fmt.Errorf("failed to find corresponding k8sService %s, error :%w ", k8s.NamespacedName(t.serviceExport), err) } @@ -275,7 +275,7 @@ func (t *svcExportTargetGroupModelBuildTask) buildTargetGroup(ctx context.Contex var ipAddressType string var err error if noSvcFoundAndDeleting { - ipAddressType = "IPV4" // just pick a default + ipAddressType = "IPV4" // Pick a default } else { ipAddressType, err = buildTargetGroupIpAddressType(svc) if err != nil { From cfeb35b495401997ff90d0b1d5103a4663fc5a9b Mon Sep 17 00:00:00 2001 From: Ryan Lymburner Date: Thu, 29 May 2025 11:07:33 -0700 Subject: [PATCH 6/8] Commit auto-generated --- .../v1alpha1/zz_generated.deepcopy.go | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/pkg/apis/applicationnetworking/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/applicationnetworking/v1alpha1/zz_generated.deepcopy.go index 26a3ef3d..7aed4fcc 100644 --- a/pkg/apis/applicationnetworking/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/applicationnetworking/v1alpha1/zz_generated.deepcopy.go @@ -132,6 +132,21 @@ func (in *ClusterStatus) DeepCopy() *ClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExportedPort) DeepCopyInto(out *ExportedPort) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExportedPort. +func (in *ExportedPort) DeepCopy() *ExportedPort { + if in == nil { + return nil + } + out := new(ExportedPort) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HealthCheckConfig) DeepCopyInto(out *HealthCheckConfig) { *out = *in @@ -303,6 +318,7 @@ func (in *ServiceExport) DeepCopyInto(out *ServiceExport) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -385,6 +401,26 @@ func (in *ServiceExportList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceExportSpec) DeepCopyInto(out *ServiceExportSpec) { + *out = *in + if in.ExportedPorts != nil { + in, out := &in.ExportedPorts, &out.ExportedPorts + *out = make([]ExportedPort, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceExportSpec. +func (in *ServiceExportSpec) DeepCopy() *ServiceExportSpec { + if in == nil { + return nil + } + out := new(ServiceExportSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceExportStatus) DeepCopyInto(out *ServiceExportStatus) { *out = *in From 1a4d6d9132379ec115ac1d1c64a26e08204bac77 Mon Sep 17 00:00:00 2001 From: Ryan Lymburner Date: Mon, 2 Jun 2025 13:25:39 -0700 Subject: [PATCH 7/8] Install latest custom CRDs on E2E tests --- .github/workflows/validate-merge-queue-e2e-test.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/validate-merge-queue-e2e-test.yaml b/.github/workflows/validate-merge-queue-e2e-test.yaml index 6892aa1e..22575a42 100644 --- a/.github/workflows/validate-merge-queue-e2e-test.yaml +++ b/.github/workflows/validate-merge-queue-e2e-test.yaml @@ -72,6 +72,16 @@ jobs: --set=image.repository=$IMAGE_REPOSITORY \ --set=image.tag=$RELEASE_VERSION \ --set=log.level=debug + - name: Install latest custom CRDs + run: | + kubectl apply -f config/crds/bases/externaldns.k8s.io_dnsendpoints.yaml + kubectl apply -f config/crds/bases/gateway.networking.k8s.io_tlsroutes.yaml + kubectl apply -f config/crds/bases/application-networking.k8s.aws_serviceexports.yaml + kubectl apply -f config/crds/bases/application-networking.k8s.aws_serviceimports.yaml + kubectl apply -f config/crds/bases/application-networking.k8s.aws_targetgrouppolicies.yaml + kubectl apply -f config/crds/bases/application-networking.k8s.aws_vpcassociationpolicies.yaml + kubectl apply -f config/crds/bases/application-networking.k8s.aws_accesslogpolicies.yaml + kubectl apply -f config/crds/bases/application-networking.k8s.aws_iamauthpolicies.yaml - name: Install Gateway API v1.2 CRDs run: | kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v1.2.0" | kubectl apply -f - From ef25684eff081eb4aad9b6b701e1d6c6316010eb Mon Sep 17 00:00:00 2001 From: Ryan Lymburner Date: Mon, 2 Jun 2025 13:31:21 -0700 Subject: [PATCH 8/8] Install latest custom CRDs on E2E tests after Gateway API CRDs --- .github/workflows/validate-merge-queue-e2e-test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate-merge-queue-e2e-test.yaml b/.github/workflows/validate-merge-queue-e2e-test.yaml index 22575a42..5f2356c6 100644 --- a/.github/workflows/validate-merge-queue-e2e-test.yaml +++ b/.github/workflows/validate-merge-queue-e2e-test.yaml @@ -72,6 +72,9 @@ jobs: --set=image.repository=$IMAGE_REPOSITORY \ --set=image.tag=$RELEASE_VERSION \ --set=log.level=debug + - name: Install Gateway API v1.2 CRDs + run: | + kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v1.2.0" | kubectl apply -f - - name: Install latest custom CRDs run: | kubectl apply -f config/crds/bases/externaldns.k8s.io_dnsendpoints.yaml @@ -82,9 +85,6 @@ jobs: kubectl apply -f config/crds/bases/application-networking.k8s.aws_vpcassociationpolicies.yaml kubectl apply -f config/crds/bases/application-networking.k8s.aws_accesslogpolicies.yaml kubectl apply -f config/crds/bases/application-networking.k8s.aws_iamauthpolicies.yaml - - name: Install Gateway API v1.2 CRDs - run: | - kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v1.2.0" | kubectl apply -f - - name: Create Lattice GatewayClass run: | kubectl apply -f files/controller-installation/gatewayclass.yaml