chore(poller): avoid warn log on shutdown (#1480)

What changed?

poller will no longer poll on permit failure
poller will no longer emit warn log on shutdown
Why?

red herring of warning log on poller acquire permit failures.

How did you test it?

Unit Test

---------

Signed-off-by: Shijie Sheng <liouvetren@gmail.com>

Author: shijiesheng

internal/internal_worker_base.go

@@ -139,7 +139,7 @@ type (
 		pollLimiter          *rate.Limiter
 		taskLimiter          *rate.Limiter
 		limiterContext       context.Context
-		limiterContextCancel func()
+		limiterContextCancel context.CancelCauseFunc
 		retrier              *backoff.ConcurrentRetrier // Service errors back off retrier
 		logger               *zap.Logger
 		metricsScope         tally.Scope
@@ -168,7 +168,7 @@ func createPollRetryPolicy() backoff.RetryPolicy {
 }
 
 func newBaseWorker(options baseWorkerOptions, logger *zap.Logger, metricsScope tally.Scope, sessionTokenBucket *sessionTokenBucket) *baseWorker {
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancelCause(context.Background())
 	logger = logger.With(zap.String(tagWorkerType, options.workerType))
 	metricsScope = tagScope(metricsScope, tagWorkerType, options.workerType)
 
@@ -324,7 +324,10 @@ func (bw *baseWorker) pollTask() {
 	if pErr := bw.concurrency.PollerPermit.Acquire(bw.limiterContext); pErr == nil {
 		defer bw.concurrency.PollerPermit.Release()
 	} else {
-		bw.logger.Warn("poller permit acquire error", zap.Error(pErr))
+		if !errors.Is(context.Cause(bw.limiterContext), errShutdown) { // don't log on shutdown
+			bw.logger.Warn("poller permit acquire error", zap.Error(pErr))
+		}
+		return // don't try to poll without a permit, will retry on next poll
 	}
 
 	bw.retrier.Throttle()
@@ -418,7 +421,7 @@ func (bw *baseWorker) Stop() {
 		return
 	}
 	close(bw.shutdownCh)
-	bw.limiterContextCancel()
+	bw.limiterContextCancel(errShutdown)
 	bw.concurrencyAutoScaler.Stop()
 
 	if success := util.AwaitWaitGroup(&bw.shutdownWG, bw.options.shutdownTimeout); !success {

internal/internal_worker_base_test.go

@@ -0,0 +1,63 @@
+package internal
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/uber-go/tally"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"go.uber.org/zap/zaptest/observer"
+
+	"go.uber.org/cadence/internal/common/debug"
+)
+
+func TestBaseWorker_pollTask_no_warnLogOnShutdown(t *testing.T) {
+	core, observed := observer.New(zapcore.InfoLevel)
+	logger := zap.New(core, zap.Development())
+	worker := newBaseWorker(baseWorkerOptions{
+		maxConcurrentTask:             1,
+		pollerCountWithoutAutoScaling: 1,
+		identity:                      "test-identity",
+		pollerTracker:                 debug.NewNoopPollerTracker(),
+		taskWorker:                    &testTaskWorker{},
+	}, logger, tally.NoopScope, nil)
+
+	// mock the worker started
+	worker.Start()
+	worker.Stop()
+	worker.pollTask()
+
+	assert.Equal(t, 0, observed.FilterMessage("poller permit acquire error").Len())
+}
+
+func TestBaseWorker_processTask_warnLogOnOtherError(t *testing.T) {
+	core, observed := observer.New(zapcore.InfoLevel)
+	logger := zap.New(core, zap.Development())
+	worker := newBaseWorker(baseWorkerOptions{
+		maxConcurrentTask:             1,
+		pollerCountWithoutAutoScaling: 1,
+		identity:                      "test-identity",
+		pollerTracker:                 debug.NewNoopPollerTracker(),
+		taskWorker:                    &testTaskWorker{},
+	}, logger, tally.NoopScope, nil)
+
+	// mock the worker started
+	worker.Start()
+	worker.limiterContextCancel(errors.New("test error"))
+	worker.pollTask()
+	worker.Stop()
+
+	assert.GreaterOrEqual(t, observed.FilterMessage("poller permit acquire error").Len(), 1)
+}
+
+type testTaskWorker struct{}
+
+func (t *testTaskWorker) PollTask() (interface{}, error) {
+	return nil, errors.New("poll in test will fail")
+}
+
+func (t *testTaskWorker) ProcessTask(task interface{}) error {
+	return nil
+}