From 7021cdaa7aa700cd04e12bd757cb0bada3243fe8 Mon Sep 17 00:00:00 2001 From: Julian Koberg Date: Thu, 11 Jun 2026 15:21:48 +0200 Subject: [PATCH] fix(deps): [OCISDEV-834] recover from dead NATS connections in nats-js-kv Bumps the kobergj/plugins nats-js-kv replace pin to fc78af4, which fixes hasConn() to treat a permanently-closed NATS connection as no connection. Previously hasConn() only checked n.conn != nil, so once the client exhausted its reconnect attempts the connection stayed non-nil but dead, initConn() never re-ran, and every KV operation failed with 'nats: connection closed' until the pod was restarted. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Julian Koberg --- .../fix-natsjskv-dead-connection.md | 18 +++++++++++++ go.mod | 2 +- go.sum | 4 +-- .../plugins/v4/store/nats-js-kv/nats.go | 26 ++++++++++++++++++- vendor/modules.txt | 4 +-- 5 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 changelog/unreleased/fix-natsjskv-dead-connection.md diff --git a/changelog/unreleased/fix-natsjskv-dead-connection.md b/changelog/unreleased/fix-natsjskv-dead-connection.md new file mode 100644 index 00000000000..9a3cd94e13f --- /dev/null +++ b/changelog/unreleased/fix-natsjskv-dead-connection.md @@ -0,0 +1,18 @@ +Bugfix: Recover from permanently-closed NATS connections in the nats-js-kv store + +The `nats-js-kv` go-micro store plugin's `hasConn()` only checked whether the +connection object was non-nil, not whether it was still alive. Once the +underlying NATS client exhausted its reconnect attempts (e.g. a NATS pod +restart longer than the client's reconnect window), the connection stayed +non-nil but permanently closed. Because connection initialization is gated on +`!hasConn()`, it never re-ran, so every subsequent KV operation failed with +`nats: connection closed` until the affected pod was restarted. + +This surfaced as several user-visible failures backed by the NATS KV cache, +e.g. all spaces becoming invisible (`storage-users` `ListStorageSpaces`) and +download failures from missing signing keys (`ocs`). + +The store plugin now treats a closed connection as no connection, so the next +operation transparently re-initializes it. + +https://github.com/owncloud/ocis/pull/12404 diff --git a/go.mod b/go.mod index af2c11015a5..693fa40be75 100644 --- a/go.mod +++ b/go.mod @@ -360,7 +360,7 @@ replace github.com/egirna/icap-client => github.com/kobergj/icap-client v0.0.0-2 replace github.com/unrolled/secure => github.com/DeepDiver1975/secure v0.0.0-20240611112133-abc838fb797c -replace github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20240807130109-f62bb67e8c90 +replace github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20260609111813-fc78af4c9138 replace go-micro.dev/v4 => github.com/kobergj/go-micro/v4 v4.0.0-20250610135441-d0b187215699 diff --git a/go.sum b/go.sum index d45e285d050..79a5b9fd17e 100644 --- a/go.sum +++ b/go.sum @@ -590,8 +590,8 @@ github.com/kobergj/gowebdav v0.0.0-20251030165916-532350997dde h1:HYcp4J4xYe2m9K github.com/kobergj/gowebdav v0.0.0-20251030165916-532350997dde/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/kobergj/icap-client v0.0.0-20250116172800-8eaa5022532b h1:NBKEgFtIukCreWOEvtrgNQusqosGyhlxzyiHbba1zEI= github.com/kobergj/icap-client v0.0.0-20250116172800-8eaa5022532b/go.mod h1:HpntrRsQA6RKNXy2Nbr4kVj+NO3OYWpAQUVxeya+3sU= -github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20240807130109-f62bb67e8c90 h1:pfI8Z5yavO6fU6vDGlWhZ4BgDlvj8c6xB7J57HfTPwA= -github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20240807130109-f62bb67e8c90/go.mod h1:pjcozWijkNPbEtX5SIQaxEW/h8VAVZYTLx+70bmB3LY= +github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20260609111813-fc78af4c9138 h1:xuwWIZj+NCv8f6EJsZon1trR463gnP9WFt8h6VgCH98= +github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20260609111813-fc78af4c9138/go.mod h1:pjcozWijkNPbEtX5SIQaxEW/h8VAVZYTLx+70bmB3LY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kovidgoyal/go-parallel v1.1.1 h1:1OzpNjtrUkBPq3UaqrnvOoB2F9RttSt811uiUXyI7ok= diff --git a/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/nats.go b/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/nats.go index f7ee7a17227..24b5e4416e2 100644 --- a/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/nats.go +++ b/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/nats.go @@ -69,6 +69,18 @@ func NewStore(opts ...store.Option) store.Store { n.setOption(opts...) + // register unset connection callback + f := n.nopts.ClosedCB + n.nopts.ClosedCB = func(c *nats.Conn) { + // reset connection + n.resetConn() + + // execute the original callback if set + if f != nil { + f(c) + } + } + return n } @@ -417,7 +429,19 @@ func (n *natsStore) hasConn() bool { n.RLock() defer n.RUnlock() - return n.conn != nil + // A non-nil conn is not enough: once the client has exhausted its + // reconnect attempts the conn is permanently closed but stays non-nil, + // which would gate initConn() off forever. Treat a closed connection as + // "no connection" so the next operation re-initializes it. + return n.conn != nil && !n.conn.IsClosed() +} + +// thread safe way to reset the connection. +func (n *natsStore) resetConn() { + n.Lock() + defer n.Unlock() + + n.conn = nil } // mustGetDefaultBucket returns the bucket with the given name creating it with default configuration if needed. diff --git a/vendor/modules.txt b/vendor/modules.txt index 183fed04970..275dc50e44b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -593,7 +593,7 @@ github.com/go-micro/plugins/v4/server/http # github.com/go-micro/plugins/v4/store/nats-js v1.2.1 ## explicit; go 1.21 github.com/go-micro/plugins/v4/store/nats-js -# github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20240726082623-6831adfdcdc4 => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20240807130109-f62bb67e8c90 +# github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20240726082623-6831adfdcdc4 => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20260609111813-fc78af4c9138 ## explicit; go 1.21 github.com/go-micro/plugins/v4/store/nats-js-kv # github.com/go-micro/plugins/v4/store/redis v1.2.1 @@ -2564,5 +2564,5 @@ stash.kopano.io/kgol/rndm # github.com/studio-b12/gowebdav => github.com/kobergj/gowebdav v0.0.0-20251030165916-532350997dde # github.com/egirna/icap-client => github.com/kobergj/icap-client v0.0.0-20250116172800-8eaa5022532b # github.com/unrolled/secure => github.com/DeepDiver1975/secure v0.0.0-20240611112133-abc838fb797c -# github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20240807130109-f62bb67e8c90 +# github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20260609111813-fc78af4c9138 # go-micro.dev/v4 => github.com/kobergj/go-micro/v4 v4.0.0-20250610135441-d0b187215699