[FEATURE] Switch to franz-go and use consumer group

.github/workflows/pr.yml

@@ -45,14 +45,12 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.23'
+          go-version: '1.24.3'
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v8
         with:
-          version: v1.61
-          skip-pkg-cache: true
-          skip-build-cache: true
+          version: v2.1.0
 
       - name: Build
         if: ${{ matrix.app != 'pkg' && matrix.app != 'internal' }}
@@ -120,13 +118,13 @@ jobs:
         working-directory: pkg
 
   approve:
-    needs: integration-tests
+    needs: [build, integration-tests]
     runs-on: ubuntu-latest
     if: ${{ always() }}
 
     steps:
-      - name: Approve Build
-        if: ${{ needs.integration-tests.result == 'success' || needs.integration-tests.result == 'skipped'}}
+      - name: Approve
+        if: ${{ needs.build.result == 'success' && needs.integration-tests.result == 'success' }}
         uses: actions/github-script@v7
         with:
           script: |
@@ -138,8 +136,8 @@ jobs:
             })
             process.exit(0)
 
-      - name: Fail Build
-        if: ${{ needs.integration-tests.result == 'failure' }}
+      - name: Reject
+        if: ${{ needs.build.result != 'success' || needs.integration-tests.result != 'success' }}
         uses: actions/github-script@v7
         with:
           script: |

.golangci.yml

@@ -1,21 +1,6 @@
-# This configuration file is not a recommendation.
-#
-# We intentionally use a limited set of linters.
-# This configuration file is used with different version of golangci-lint to avoid regressions:
-# the linters can change between version,
-# their configuration may be not compatible or their reports can be different,
-# and this can break some of our tests.
-# Also, some linters are not relevant for the project (e.g. linters related to SQL).
-#
-# We have specific constraints, so we use a specific configuration.
-#
-# See the file `.golangci.reference.yml` to have a list of all available configuration options.
-
+version: '2'
 linters:
-  disable-all: true
-  # This list of linters is not a recommendation (same thing for all this configuration file).
-  # We intentionally use a limited set of linters.
-  # See the comment on top of this file.
+  default: none
   enable:
     - bodyclose
     - copyloopvar
@@ -26,39 +11,51 @@ linters:
     - funlen
     - gocheckcompilerdirectives
     - gochecknoinits
-    - gochecknoinits
     - goconst
     - gocritic
     - gocyclo
-    - godox
     - godot
-    - gofmt
-    - goimports
-    - mnd
+    - godox
     - goprintffuncname
     - gosec
-    - gosimple
     - govet
-    - intrange
     - ineffassign
+    - intrange
     - lll
     - misspell
+    - mnd
     - nakedret
     - noctx
     - nolintlint
     - revive
     - staticcheck
-    - stylecheck
     - testifylint
     - unconvert
     - unparam
     - unused
     - whitespace
     - wrapcheck
-
-linters-settings:
-  funlen:
-    lines: 199
-
-run:
-  timeout: 5m
+  settings:
+    funlen:
+      lines: 199
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

cmd/flight-processor/main.go

@@ -131,15 +131,12 @@ func startBackgroundJobs(ctx context.Context, processor *service.Processor) erro
 func safeShutDown(ctx context.Context, processor *service.Processor, mongodb *mongo.Client) error {
 	slog.Info("Shutting down components")
 
-	if err := processor.MessageReader.Close(); err != nil {
-		slog.Error("Failed to shutdown message reader", "error", err)
-		return fmt.Errorf("failed to shutdown message reader: %w", err)
-	}
-
 	if err := mongodb.Client.Disconnect(ctx); err != nil {
 		slog.Error("Failed to shutdown MongoDB client", "error", err)
 		return fmt.Errorf("failed to shutdown mongodb client: %w", err)
 	}
 
+	processor.MessageReader.Close()
+
 	return nil
 }

cmd/flight-reader/main.go

@@ -155,14 +155,8 @@ func safeShutDown(
 		return fmt.Errorf("failed to shutdown HTTP server: %w", err)
 	}
 
-	// Close the HTTP client
 	httpClient.CloseIdleConnections()
+	reader.Close()
 
-	if err := reader.Close(); err != nil {
-		slog.Error("Failed to shutdown reader", "error", err)
-		return fmt.Errorf("failed to shutdown reader: %w", err)
-	}
-
-	// Return nil if all shutdowns were successful
 	return nil
 }

go.mod

@@ -1,10 +1,10 @@
 module github.com/ansoncht/flight-microservices
 
-go 1.23.3
+go 1.24.3
 
 require (
-	golang.org/x/sync v0.13.0
-	google.golang.org/protobuf v1.34.2
+	golang.org/x/sync v0.14.0
+	google.golang.org/protobuf v1.36.5
 )
 
 require (
@@ -29,8 +29,9 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.10 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -45,7 +46,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
-	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
@@ -58,9 +59,9 @@ require (
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/testcontainers/testcontainers-go/modules/kafka v0.37.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/twmb/franz-go/pkg/kmsg v1.11.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
@@ -73,25 +74,27 @@ require (
 	go.opentelemetry.io/otel/trace v1.35.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.37.0 // indirect
+	golang.org/x/crypto v0.38.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/mod v0.18.0 // indirect
 	golang.org/x/net v0.39.0 // indirect
-	golang.org/x/sys v0.32.0 // indirect
-	golang.org/x/text v0.24.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
 	github.com/michimani/gotwi v0.16.1
-	github.com/segmentio/kafka-go v0.4.47
 	github.com/spf13/viper v1.19.0
 	github.com/stretchr/testify v1.10.0
 	github.com/testcontainers/testcontainers-go v0.37.0
+	github.com/testcontainers/testcontainers-go/modules/kafka v0.37.0
 	github.com/testcontainers/testcontainers-go/modules/mongodb v0.37.0
+	github.com/twmb/franz-go v1.19.1
+	github.com/twmb/franz-go/pkg/kadm v1.16.0
 	go.mongodb.org/mongo-driver v1.17.1
 	go.uber.org/mock v0.5.2
-	google.golang.org/grpc v1.68.0
+	google.golang.org/grpc v1.70.0
 )