diff --git a/checkstyle/import-control-server.xml b/checkstyle/import-control-server.xml index 1f020ebe71748..08fa0bfdfa51f 100644 --- a/checkstyle/import-control-server.xml +++ b/checkstyle/import-control-server.xml @@ -116,6 +116,7 @@ + diff --git a/core/src/main/java/kafka/server/ClientRequestQuotaManager.java b/core/src/main/java/kafka/server/ClientRequestQuotaManager.java index 3f2398b8358f5..e685b6aa50a94 100644 --- a/core/src/main/java/kafka/server/ClientRequestQuotaManager.java +++ b/core/src/main/java/kafka/server/ClientRequestQuotaManager.java @@ -16,8 +16,6 @@ */ package kafka.server; -import kafka.network.RequestChannel; - import org.apache.kafka.common.MetricName; import org.apache.kafka.common.internals.Plugin; import org.apache.kafka.common.metrics.Metrics; @@ -25,6 +23,7 @@ import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Rate; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.network.Request; import org.apache.kafka.server.config.ClientQuotaManagerConfig; import org.apache.kafka.server.quota.ClientQuotaCallback; import org.apache.kafka.server.quota.ClientQuotaManager; @@ -76,22 +75,22 @@ private void recordExempt(double value) { * @param request client request * @return Number of milliseconds to throttle in case of quota violation. Zero otherwise */ - public int maybeRecordAndGetThrottleTimeMs(RequestChannel.Request request, long timeMs) { + public int maybeRecordAndGetThrottleTimeMs(Request request, long timeMs) { if (quotasEnabled()) { - request.setRecordNetworkThreadTimeCallback(timeNanos -> { - recordNoThrottle(request.session(), request.header().clientId(), nanosToPercentage(Long.parseLong(timeNanos.toString()))); - }); + request.setRecordNetworkThreadTimeCallback(timeNanos -> + recordNoThrottle(request.session(), request.header().clientId(), nanosToPercentage(timeNanos)) + ); return recordAndGetThrottleTimeMs(request.session(), request.header().clientId(), nanosToPercentage(request.requestThreadTimeNanos()), timeMs); } else { return 0; } } - public void maybeRecordExempt(RequestChannel.Request request) { + public void maybeRecordExempt(Request request) { if (quotasEnabled()) { - request.setRecordNetworkThreadTimeCallback(timeNanos -> { - recordExempt(nanosToPercentage(Long.parseLong(timeNanos.toString()))); - }); + request.setRecordNetworkThreadTimeCallback(timeNanos -> + recordExempt(nanosToPercentage(timeNanos)) + ); recordExempt(nanosToPercentage(request.requestThreadTimeNanos())); } } diff --git a/core/src/main/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandler.java b/core/src/main/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandler.java index 9b6a04ee979eb..101f09c2a7102 100644 --- a/core/src/main/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandler.java +++ b/core/src/main/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandler.java @@ -17,7 +17,6 @@ package kafka.server.handlers; -import kafka.network.RequestChannel; import kafka.server.AuthHelper; import kafka.server.KafkaConfig; @@ -31,6 +30,7 @@ import org.apache.kafka.common.requests.DescribeTopicPartitionsRequest; import org.apache.kafka.common.resource.Resource; import org.apache.kafka.metadata.MetadataCache; +import org.apache.kafka.network.Request; import java.util.HashSet; import java.util.List; @@ -55,7 +55,7 @@ public DescribeTopicPartitionsRequestHandler( this.config = config; } - public DescribeTopicPartitionsResponseData handleDescribeTopicPartitionsRequest(RequestChannel.Request abstractRequest) { + public DescribeTopicPartitionsResponseData handleDescribeTopicPartitionsRequest(Request abstractRequest) { DescribeTopicPartitionsRequestData request = ((DescribeTopicPartitionsRequest) abstractRequest.loggableRequest()).data(); Set topics = new HashSet<>(); boolean fetchAllTopics = request.topics().isEmpty(); diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index d3e99cbba43a0..01b281c4797da 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -17,287 +17,28 @@ package kafka.network -import java.nio.ByteBuffer import java.util.concurrent._ import com.fasterxml.jackson.databind.JsonNode -import com.typesafe.scalalogging.Logger -import kafka.network import kafka.utils.Logging -import org.apache.kafka.common.config.ConfigResource -import org.apache.kafka.common.memory.MemoryPool -import org.apache.kafka.common.message.EnvelopeResponseData import org.apache.kafka.common.network.Send import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests._ import org.apache.kafka.common.utils.Time -import org.apache.kafka.network.metrics.{RequestChannelMetrics, RequestMetrics} -import org.apache.kafka.server.common.RequestLocal import org.apache.kafka.common.metrics.internals.MetricsUtils +import org.apache.kafka.network.{BaseRequest, CallbackRequest, Request, ShutdownRequest, WakeupRequest} +import org.apache.kafka.network.metrics.RequestChannelMetrics import org.apache.kafka.server.metrics.KafkaMetricsGroup -import org.apache.kafka.network.{RequestConvertToJson, Session} -import org.apache.kafka.server.config.AbstractKafkaConfig +import java.util.OptionalLong import scala.jdk.CollectionConverters._ -import scala.jdk.OptionConverters.RichOption -import scala.reflect.ClassTag +import scala.jdk.OptionConverters._ object RequestChannel extends Logging { - private val requestLogger = Logger("kafka.request.logger") private val RequestQueueSizeMetric = "RequestQueueSize" private val ResponseQueueSizeMetric = "ResponseQueueSize" val ProcessorMetricTag = "processor" - /** - * Deprecated protocol apis are logged at info level while the rest are logged at debug level. - * That makes it possible to enable the former without enabling latter. - */ - private def isRequestLoggingEnabled(header: RequestHeader): Boolean = requestLogger.underlying.isDebugEnabled || - (requestLogger.underlying.isInfoEnabled && header.isApiVersionDeprecated()) - - sealed trait BaseRequest - case object ShutdownRequest extends BaseRequest - case object WakeupRequest extends BaseRequest - - case class CallbackRequest(fun: RequestLocal => Unit, - originalRequest: Request) extends BaseRequest - - class Request(val processor: Int, - val context: RequestContext, - val startTimeNanos: Long, - val memoryPool: MemoryPool, - @volatile var buffer: ByteBuffer, - metrics: RequestChannelMetrics, - val envelope: Option[RequestChannel.Request] = None) extends BaseRequest { - // These need to be volatile because the readers are in the network thread and the writers are in the request - // handler threads or the purgatory threads - @volatile var requestDequeueTimeNanos: Long = -1L - @volatile var apiLocalCompleteTimeNanos: Long = -1L - @volatile var responseCompleteTimeNanos: Long = -1L - @volatile var responseDequeueTimeNanos: Long = -1L - @volatile var messageConversionsTimeNanos: Long = 0L - @volatile var apiThrottleTimeMs: Long = 0L - @volatile var temporaryMemoryBytes: Long = 0L - @volatile var recordNetworkThreadTimeCallback: Option[java.util.function.Consumer[java.lang.Long]] = None - @volatile var callbackRequestDequeueTimeNanos: Option[Long] = None - @volatile var callbackRequestCompleteTimeNanos: Option[Long] = None - - val session: Session = new Session(context.principal, context.clientAddress) - - private val bodyAndSize: RequestAndSize = context.parseRequest(buffer) - - // This is constructed on creation of a Request so that the JSON representation is computed before the request is - // processed by the api layer. Otherwise, a ProduceRequest can occur without its data (ie. it goes into purgatory). - val requestLog: Option[JsonNode] = - if (RequestChannel.isRequestLoggingEnabled(context.header)) Some(RequestConvertToJson.request(loggableRequest)) - else None - - def header: RequestHeader = context.header - - private def sizeOfBodyInBytes: Int = bodyAndSize.size - - def sizeInBytes: Int = header.size + sizeOfBodyInBytes - - //most request types are parsed entirely into objects at this point. for those we can release the underlying buffer. - //some (like produce, or any time the schema contains fields of types BYTES or NULLABLE_BYTES) retain a reference - //to the buffer. for those requests we cannot release the buffer early, but only when request processing is done. - if (!header.apiKey.requiresDelayedAllocation) { - releaseBuffer() - } - - def isForwarded: Boolean = envelope.isDefined - - private def shouldReturnNotController(response: AbstractResponse): Boolean = { - response match { - case _: DescribeQuorumResponse => response.errorCounts.containsKey(Errors.NOT_LEADER_OR_FOLLOWER) - case _ => response.errorCounts.containsKey(Errors.NOT_CONTROLLER) - } - } - - def buildResponseSend(abstractResponse: AbstractResponse): Send = { - envelope match { - case Some(request) => - val envelopeResponse = if (shouldReturnNotController(abstractResponse)) { - // Since it's a NOT_CONTROLLER error response, we need to make envelope response with NOT_CONTROLLER error - // to notify the requester (i.e. NodeToControllerRequestThread) to update active controller - new EnvelopeResponse(new EnvelopeResponseData() - .setErrorCode(Errors.NOT_CONTROLLER.code())) - } else { - val responseBytes = context.buildResponseEnvelopePayload(abstractResponse) - new EnvelopeResponse(responseBytes, Errors.NONE) - } - request.context.buildResponseSend(envelopeResponse) - case None => - context.buildResponseSend(abstractResponse) - } - } - - def responseNode(response: AbstractResponse): Option[JsonNode] = { - if (RequestChannel.isRequestLoggingEnabled(context.header)) - Some(RequestConvertToJson.response(response, context.apiVersion)) - else - None - } - - def headerForLoggingOrThrottling(): RequestHeader = { - envelope match { - case Some(request) => - request.context.header - case None => - context.header - } - } - - def requestDesc(details: Boolean): String = { - val forwardDescription = envelope.map { request => - s"Forwarded request: ${request.context} " - }.getOrElse("") - s"$forwardDescription$header -- ${loggableRequest.toString(details)}" - } - - def body[T <: AbstractRequest](implicit classTag: ClassTag[T]): T = { - bodyAndSize.request match { - case r: T => r - case r => - throw new ClassCastException(s"Expected request with type ${classTag.runtimeClass}, but found ${r.getClass}") - } - } - - def loggableRequest: AbstractRequest = { - - bodyAndSize.request match { - case alterConfigs: AlterConfigsRequest => - val newData = alterConfigs.data().duplicate() - newData.resources().forEach(resource => { - val resourceType = ConfigResource.Type.forId(resource.resourceType()) - resource.configs().forEach(config => { - config.setValue(AbstractKafkaConfig.loggableValue(resourceType, config.name(), config.value())) - }) - }) - new AlterConfigsRequest(newData, alterConfigs.version()) - - case alterConfigs: IncrementalAlterConfigsRequest => - val newData = alterConfigs.data().duplicate() - newData.resources().forEach(resource => { - val resourceType = ConfigResource.Type.forId(resource.resourceType()) - resource.configs().forEach(config => { - config.setValue(AbstractKafkaConfig.loggableValue(resourceType, config.name(), config.value())) - }) - }) - new IncrementalAlterConfigsRequest.Builder(newData).build(alterConfigs.version()) - - case _ => - bodyAndSize.request - } - } - - trace(s"Processor $processor received request: ${requestDesc(true)}") - - def requestThreadTimeNanos: Long = { - if (apiLocalCompleteTimeNanos == -1L) apiLocalCompleteTimeNanos = Time.SYSTEM.nanoseconds - math.max(apiLocalCompleteTimeNanos - requestDequeueTimeNanos, 0L) - } - - def updateRequestMetrics(networkThreadTimeNanos: Long, response: Response): Unit = { - val endTimeNanos = Time.SYSTEM.nanoseconds - - /** - * Converts nanos to millis with micros precision as additional decimal places in the request log have low - * signal to noise ratio. When it comes to metrics, there is little difference either way as we round the value - * to the nearest long. - */ - def nanosToMs(nanos: Long): Double = { - val positiveNanos = math.max(nanos, 0) - TimeUnit.NANOSECONDS.toMicros(positiveNanos).toDouble / TimeUnit.MILLISECONDS.toMicros(1) - } - - val requestQueueTimeMs = nanosToMs(requestDequeueTimeNanos - startTimeNanos) - val callbackRequestTimeNanos = callbackRequestCompleteTimeNanos.getOrElse(0L) - callbackRequestDequeueTimeNanos.getOrElse(0L) - val apiLocalTimeMs = nanosToMs(apiLocalCompleteTimeNanos - requestDequeueTimeNanos + callbackRequestTimeNanos) - val apiRemoteTimeMs = nanosToMs(responseCompleteTimeNanos - apiLocalCompleteTimeNanos - callbackRequestTimeNanos) - val responseQueueTimeMs = nanosToMs(responseDequeueTimeNanos - responseCompleteTimeNanos) - val responseSendTimeMs = nanosToMs(endTimeNanos - responseDequeueTimeNanos) - val messageConversionsTimeMs = nanosToMs(messageConversionsTimeNanos) - val totalTimeMs = nanosToMs(endTimeNanos - startTimeNanos) - val overrideMetricNames = - if (header.apiKey == ApiKeys.FETCH) { - val specifiedMetricName = - if (body[FetchRequest].isFromFollower) RequestMetrics.FOLLOW_FETCH_METRIC_NAME - else RequestMetrics.CONSUMER_FETCH_METRIC_NAME - Seq(specifiedMetricName, header.apiKey.name) - } else if (header.apiKey == ApiKeys.ADD_PARTITIONS_TO_TXN && body[AddPartitionsToTxnRequest].allVerifyOnlyRequest) { - Seq(RequestMetrics.VERIFY_PARTITIONS_IN_TXN_METRIC_NAME) - } else if (header.apiKey == ApiKeys.LIST_CONFIG_RESOURCES && header.apiVersion == 0) { - Seq(RequestMetrics.LIST_CLIENT_METRICS_RESOURCES_METRIC_NAME, header.apiKey.name) - } else { - Seq(header.apiKey.name) - } - overrideMetricNames.foreach { metricName => - val m = metrics(metricName) - m.requestRate(header.apiVersion).mark() - m.deprecatedRequestRate(header.apiKey, header.apiVersion, context.clientInformation).ifPresent(_.mark()) - m.requestQueueTimeHist.update(Math.round(requestQueueTimeMs)) - m.localTimeHist.update(Math.round(apiLocalTimeMs)) - m.remoteTimeHist.update(Math.round(apiRemoteTimeMs)) - m.throttleTimeHist.update(apiThrottleTimeMs) - m.responseQueueTimeHist.update(Math.round(responseQueueTimeMs)) - m.responseSendTimeHist.update(Math.round(responseSendTimeMs)) - m.totalTimeHist.update(Math.round(totalTimeMs)) - m.requestBytesHist.update(sizeOfBodyInBytes) - m.messageConversionsTimeHist.ifPresent(_.update(Math.round(messageConversionsTimeMs))) - m.tempMemoryBytesHist.ifPresent(_.update(temporaryMemoryBytes)) - } - - // Records network handler thread usage. This is included towards the request quota for the - // user/client. Throttling is only performed when request handler thread usage - // is recorded, just before responses are queued for delivery. - // The time recorded here is the time spent on the network thread for receiving this request - // and sending the response. Note that for the first request on a connection, the time includes - // the total time spent on authentication, which may be significant for SASL/SSL. - recordNetworkThreadTimeCallback.foreach(record => record.accept(networkThreadTimeNanos)) - - if (isRequestLoggingEnabled(header)) { - val desc = RequestConvertToJson.requestDescMetrics(header, requestLog.toJava, response.responseLog.toJava, - context, session, isForwarded, - totalTimeMs, requestQueueTimeMs, apiLocalTimeMs, - apiRemoteTimeMs, apiThrottleTimeMs, responseQueueTimeMs, - responseSendTimeMs, temporaryMemoryBytes, - messageConversionsTimeMs) - val logPrefix = "Completed request:{}" - // log deprecated apis at `info` level to allow them to be selectively enabled - if (header.isApiVersionDeprecated()) - requestLogger.info(logPrefix, desc) - else - requestLogger.debug(logPrefix, desc) - } - } - - def releaseBuffer(): Unit = { - envelope match { - case Some(request) => - request.releaseBuffer() - case None => - if (buffer != null) { - memoryPool.release(buffer) - buffer = null - } - } - } - - def setRecordNetworkThreadTimeCallback(callback: java.util.function.Consumer[java.lang.Long]): Unit = { - recordNetworkThreadTimeCallback = Some(callback) - } - - override def toString: String = s"Request(processor=$processor, " + - s"connectionId=${context.connectionId}, " + - s"session=$session, " + - s"listenerName=${context.listenerName}, " + - s"securityProtocol=${context.securityProtocol}, " + - s"buffer=$buffer, " + - s"envelope=$envelope)" - - } - sealed abstract class Response(val request: Request) { def processor: Int = request.processor @@ -378,12 +119,12 @@ class RequestChannel(val queueSize: Int, } /** Send a request to be handled, potentially blocking until there is room in the queue for the request */ - def sendRequest(request: RequestChannel.Request): Unit = { + def sendRequest(request: Request): Unit = { requestQueue.put(request) } def closeConnection( - request: RequestChannel.Request, + request: Request, errorCounts: java.util.Map[Errors, Integer] ): Unit = { // This case is used when the request handler has encountered an error, but the client @@ -393,7 +134,7 @@ class RequestChannel(val queueSize: Int, } def sendResponse( - request: RequestChannel.Request, + request: Request, response: AbstractResponse, onComplete: Option[Send => Unit] ): Unit = { @@ -401,20 +142,20 @@ class RequestChannel(val queueSize: Int, sendResponse(new RequestChannel.SendResponse( request, request.buildResponseSend(response), - request.responseNode(response), + request.responseNode(response).toScala, onComplete )) } - def sendNoOpResponse(request: RequestChannel.Request): Unit = { - sendResponse(new network.RequestChannel.NoOpResponse(request)) + def sendNoOpResponse(request: Request): Unit = { + sendResponse(new RequestChannel.NoOpResponse(request)) } - def startThrottling(request: RequestChannel.Request): Unit = { + def startThrottling(request: Request): Unit = { sendResponse(new RequestChannel.StartThrottlingResponse(request)) } - def endThrottling(request: RequestChannel.Request): Unit = { + def endThrottling(request: Request): Unit = { sendResponse(new EndThrottlingResponse(request)) } @@ -442,12 +183,12 @@ class RequestChannel(val queueSize: Int, case _: SendResponse | _: NoOpResponse | _: CloseConnectionResponse => val request = response.request val timeNanos = time.nanoseconds() - request.responseCompleteTimeNanos = timeNanos + request.responseCompleteTimeNanos(timeNanos) if (request.apiLocalCompleteTimeNanos == -1L) - request.apiLocalCompleteTimeNanos = timeNanos + request.apiLocalCompleteTimeNanos(timeNanos) // If this callback was executed after KafkaApis returned we will need to adjust the callback completion time here. - if (request.callbackRequestDequeueTimeNanos.isDefined && request.callbackRequestCompleteTimeNanos.isEmpty) - request.callbackRequestCompleteTimeNanos = Some(time.nanoseconds()) + if (request.callbackRequestDequeueTimeNanos.isPresent && request.callbackRequestCompleteTimeNanos.isEmpty) + request.callbackRequestCompleteTimeNanos(OptionalLong.of(time.nanoseconds())) // For a given request, these may happen in addition to one in the previous section, skip updating the metrics case _: StartThrottlingResponse | _: EndThrottlingResponse => () } @@ -463,21 +204,21 @@ class RequestChannel(val queueSize: Int, /** Get the next request or block until specified time has elapsed * Check the callback queue and execute first if present since these * requests have already waited in line. */ - def receiveRequest(timeout: Long): RequestChannel.BaseRequest = { + def receiveRequest(timeout: Long): BaseRequest = { val callbackRequest = callbackQueue.poll() if (callbackRequest != null) callbackRequest else { val request = requestQueue.poll(timeout, TimeUnit.MILLISECONDS) request match { - case WakeupRequest => callbackQueue.poll() + case _: WakeupRequest => callbackQueue.poll() case _ => request } } } /** Get the next request or block until there is one */ - def receiveRequest(): RequestChannel.BaseRequest = + def receiveRequest(): BaseRequest = requestQueue.take() def updateErrorMetrics(apiKey: ApiKeys, errors: collection.Map[Errors, Integer]): Unit = { @@ -496,11 +237,11 @@ class RequestChannel(val queueSize: Int, metrics.close() } - def sendShutdownRequest(): Unit = requestQueue.put(ShutdownRequest) + def sendShutdownRequest(): Unit = requestQueue.put(ShutdownRequest.INSTANCE) def sendCallbackRequest(request: CallbackRequest): Unit = { callbackQueue.put(request) - if (!requestQueue.offer(RequestChannel.WakeupRequest)) + if (!requestQueue.offer(WakeupRequest.INSTANCE)) trace("Wakeup request could not be added to queue. This means queue is full, so we will still process callback.") } diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 7aa170fb17817..f1658b5d647bb 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -43,7 +43,7 @@ import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.{Time, Utils} import org.apache.kafka.common.utils.internals.LogContext import org.apache.kafka.common.{Endpoint, KafkaException, MetricName, Reconfigurable} -import org.apache.kafka.network.{ConnectionQuotaEntity, ConnectionThrottledException, SocketServer => JSocketServer, SocketServerConfigs, TooManyConnectionsException} +import org.apache.kafka.network.{ConnectionQuotaEntity, ConnectionThrottledException, Request, SocketServer => JSocketServer, SocketServerConfigs, TooManyConnectionsException} import org.apache.kafka.security.CredentialProvider import org.apache.kafka.server.{ApiVersionManager, ServerSocketFactory} import org.apache.kafka.server.config.QuotaConfig @@ -58,6 +58,7 @@ import org.slf4j.event.Level import scala.collection._ import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters._ +import scala.jdk.OptionConverters._ import scala.util.control.ControlThrowable /** @@ -980,7 +981,7 @@ private[kafka] class Processor( // `channel` can be None if the connection was closed remotely or if selector closed it for being idle for too long if (channel(connectionId).isEmpty) { warn(s"Attempting to send response via channel for which there is no open connection, connection id $connectionId") - response.request.updateRequestMetrics(0L, response) + response.request.updateRequestMetrics(0L, response.responseLog.toJava) } // Invoke send for closingChannel as well so that the send is failed and the channel closed properly and // removed from the Selector after discarding any pending staged receives. @@ -1005,7 +1006,7 @@ private[kafka] class Processor( private def processCompletedReceives(): Unit = { selector.completedReceives.forEach { receive => var header: RequestHeader = null - var req: RequestChannel.Request = null + var req: Request = null try { openOrClosingChannel(receive.source) match { case Some(channel) => @@ -1027,13 +1028,12 @@ private[kafka] class Processor( channel.principal, listenerName, securityProtocol, channel.channelMetadataRegistry.clientInformation, isPrivilegedListener, channel.principalSerde) - req = new RequestChannel.Request(processor = id, context = context, - startTimeNanos = nowNanos, memoryPool, receive.payload, requestChannel.metrics, None) + req = new Request(id, context, nowNanos, memoryPool, receive.payload, requestChannel.metrics) // KIP-511: ApiVersionsRequest is intercepted here to catch the client software name // and version. It is done here to avoid wiring things up to the api layer. if (header.apiKey == ApiKeys.API_VERSIONS) { - val apiVersionsRequest = req.body[ApiVersionsRequest] + val apiVersionsRequest = req.body(classOf[ApiVersionsRequest]) if (apiVersionsRequest.isValid) { channel.channelMetadataRegistry.registerClientInformation(new ClientInformation( apiVersionsRequest.data.clientSoftwareName, @@ -1090,7 +1090,7 @@ private[kafka] class Processor( private def updateRequestMetrics(response: RequestChannel.Response): Unit = { val request = response.request val networkThreadTimeNanos = openOrClosingChannel(request.context.connectionId).fold(0L)(_.getAndResetNetworkThreadTimeNanos()) - request.updateRequestMetrics(networkThreadTimeNanos, response) + request.updateRequestMetrics(networkThreadTimeNanos, response.responseLog.toJava) } private def processDisconnected(): Unit = { @@ -1217,7 +1217,7 @@ private[kafka] class Processor( private def dequeueResponse(): RequestChannel.Response = { val response = responseQueue.poll() if (response != null) - response.request.responseDequeueTimeNanos = Time.SYSTEM.nanoseconds + response.request.responseDequeueTimeNanos(Time.SYSTEM.nanoseconds) response } diff --git a/core/src/main/scala/kafka/server/AclApis.scala b/core/src/main/scala/kafka/server/AclApis.scala index 549180dc030f4..10c7f6bb4d70d 100644 --- a/core/src/main/scala/kafka/server/AclApis.scala +++ b/core/src/main/scala/kafka/server/AclApis.scala @@ -17,7 +17,6 @@ package kafka.server -import kafka.network.RequestChannel import kafka.utils.Logging import org.apache.kafka.common.acl.AclOperation._ import org.apache.kafka.common.acl.AclBinding @@ -30,6 +29,7 @@ import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests._ import org.apache.kafka.common.resource.Resource.CLUSTER_NAME import org.apache.kafka.common.resource.ResourceType +import org.apache.kafka.network.Request import org.apache.kafka.security.authorizer.AuthorizerUtils import org.apache.kafka.server.ProcessRole import org.apache.kafka.server.authorizer._ @@ -59,9 +59,9 @@ class AclApis(authHelper: AuthHelper, def close(): Unit = alterAclsPurgatory.shutdown() - def handleDescribeAcls(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleDescribeAcls(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, DESCRIBE) - val describeAclsRequest = request.body[DescribeAclsRequest] + val describeAclsRequest = request.body(classOf[DescribeAclsRequest]) authorizerPlugin match { case None => requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => @@ -81,9 +81,9 @@ class AclApis(authHelper: AuthHelper, CompletableFuture.completedFuture[Unit](()) } - def handleCreateAcls(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleCreateAcls(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, ALTER) - val createAclsRequest = request.body[CreateAclsRequest] + val createAclsRequest = request.body(classOf[CreateAclsRequest]) authorizerPlugin match { case None => requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => @@ -137,9 +137,9 @@ class AclApis(authHelper: AuthHelper, } } - def handleDeleteAcls(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleDeleteAcls(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, ALTER) - val deleteAclsRequest = request.body[DeleteAclsRequest] + val deleteAclsRequest = request.body(classOf[DeleteAclsRequest]) authorizerPlugin match { case None => requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => diff --git a/core/src/main/scala/kafka/server/AuthHelper.scala b/core/src/main/scala/kafka/server/AuthHelper.scala index 60140a14dc2d2..ea76650da68f3 100644 --- a/core/src/main/scala/kafka/server/AuthHelper.scala +++ b/core/src/main/scala/kafka/server/AuthHelper.scala @@ -19,7 +19,6 @@ package kafka.server import java.lang.{Byte => JByte} import java.util.Collections -import kafka.network.RequestChannel import org.apache.kafka.clients.admin.EndpointType import org.apache.kafka.common.acl.AclOperation import org.apache.kafka.common.acl.AclOperation.DESCRIBE @@ -27,6 +26,7 @@ import org.apache.kafka.common.errors.ClusterAuthorizationException import org.apache.kafka.common.internals.Plugin import org.apache.kafka.common.message.DescribeClusterResponseData import org.apache.kafka.common.message.DescribeClusterResponseData.DescribeClusterBrokerCollection +import org.apache.kafka.network.Request import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests.{DescribeClusterRequest, RequestContext} import org.apache.kafka.common.resource.Resource.CLUSTER_NAME @@ -54,12 +54,12 @@ class AuthHelper(authorizer: Option[Plugin[Authorizer]]) { } } - def authorizeClusterOperation(request: RequestChannel.Request, operation: AclOperation): Unit = { + def authorizeClusterOperation(request: Request, operation: AclOperation): Unit = { if (!authorize(request.context, operation, CLUSTER, CLUSTER_NAME)) throw new ClusterAuthorizationException(s"Request $request needs $operation permission.") } - def authorizedOperations(request: RequestChannel.Request, resource: Resource): Int = { + def authorizedOperations(request: Request, resource: Resource): Int = { val supportedOps = AclEntry.supportedOperations(resource.resourceType).asScala.toList val authorizedOps = authorizer match { case Some(authZ) => @@ -120,13 +120,13 @@ class AuthHelper(authorizer: Option[Plugin[Authorizer]]) { } def computeDescribeClusterResponse( - request: RequestChannel.Request, + request: Request, expectedEndpointType: EndpointType, clusterId: String, getNodes: () => DescribeClusterBrokerCollection, getControllerId: () => Int ): DescribeClusterResponseData = { - val describeClusterRequest = request.body[DescribeClusterRequest] + val describeClusterRequest = request.body(classOf[DescribeClusterRequest]) val requestEndpointType = EndpointType.fromId(describeClusterRequest.data().endpointType()) if (requestEndpointType.equals(EndpointType.UNKNOWN)) { return new DescribeClusterResponseData(). diff --git a/core/src/main/scala/kafka/server/ConfigHelper.scala b/core/src/main/scala/kafka/server/ConfigHelper.scala index 2f7a86b4edac2..304d8c155411a 100644 --- a/core/src/main/scala/kafka/server/ConfigHelper.scala +++ b/core/src/main/scala/kafka/server/ConfigHelper.scala @@ -17,8 +17,6 @@ package kafka.server -import kafka.network.RequestChannel - import java.util.{Collections, Properties} import kafka.utils.Logging import org.apache.kafka.common.acl.AclOperation.DESCRIBE_CONFIGS @@ -34,6 +32,7 @@ import org.apache.kafka.common.resource.Resource.CLUSTER_NAME import org.apache.kafka.common.resource.ResourceType.{CLUSTER, GROUP, TOPIC} import org.apache.kafka.coordinator.group.GroupConfig import org.apache.kafka.metadata.{ConfigRepository, MetadataCache} +import org.apache.kafka.network.Request import org.apache.kafka.server.ConfigHelperUtils.createResponseConfig import org.apache.kafka.server.config.{DynamicBrokerConfig, ServerTopicConfigSynonyms} import org.apache.kafka.server.logger.LoggingController @@ -47,10 +46,10 @@ import scala.jdk.OptionConverters.RichOptional class ConfigHelper(metadataCache: MetadataCache, config: KafkaConfig, configRepository: ConfigRepository) extends Logging { def handleDescribeConfigsRequest( - request: RequestChannel.Request, + request: Request, authHelper: AuthHelper ): DescribeConfigsResponseData = { - val describeConfigsRequest = request.body[DescribeConfigsRequest] + val describeConfigsRequest = request.body(classOf[DescribeConfigsRequest]) val (authorizedResources, unauthorizedResources) = describeConfigsRequest.data.resources.asScala.partition { resource => ConfigResource.Type.forId(resource.resourceType) match { case ConfigResource.Type.BROKER | ConfigResource.Type.BROKER_LOGGER | ConfigResource.Type.CLIENT_METRICS => diff --git a/core/src/main/scala/kafka/server/ControllerApis.scala b/core/src/main/scala/kafka/server/ControllerApis.scala index fbe7601250892..ef82b0cfb843c 100644 --- a/core/src/main/scala/kafka/server/ControllerApis.scala +++ b/core/src/main/scala/kafka/server/ControllerApis.scala @@ -45,14 +45,14 @@ import org.apache.kafka.common.protocol.{ApiKeys, ApiMessage, Errors} import org.apache.kafka.common.requests._ import org.apache.kafka.common.resource.Resource.CLUSTER_NAME import org.apache.kafka.common.resource.ResourceType.{CLUSTER, GROUP, TOPIC, USER} +import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} import org.apache.kafka.common.utils.Time import org.apache.kafka.common.Uuid import org.apache.kafka.controller.ControllerRequestContext.requestTimeoutMsToDeadlineNs import org.apache.kafka.controller.{Controller, ControllerRequestContext} import org.apache.kafka.image.publisher.ControllerRegistrationsPublisher import org.apache.kafka.metadata.{BrokerHeartbeatReply, BrokerRegistrationReply, KRaftMetadataCache} -import org.apache.kafka.common.security.auth.KafkaPrincipal -import org.apache.kafka.common.security.auth.SecurityProtocol +import org.apache.kafka.network.Request import org.apache.kafka.raft.RaftManager import org.apache.kafka.security.DelegationTokenManager import org.apache.kafka.server.{ApiVersionManager, ProcessRole} @@ -91,7 +91,7 @@ class ControllerApis( def close(): Unit = aclApis.close() - override def handle(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { + override def handle(request: Request, requestLocal: RequestLocal): Unit = { try { val handlerFuture: CompletableFuture[Unit] = request.header.apiKey match { case ApiKeys.FETCH => handleFetch(request) @@ -156,12 +156,12 @@ class ControllerApis( } finally { // Only record local completion time if it is unset. if (request.apiLocalCompleteTimeNanos < 0) { - request.apiLocalCompleteTimeNanos = time.nanoseconds + request.apiLocalCompleteTimeNanos(time.nanoseconds) } } } - def handleEnvelopeRequest(request: RequestChannel.Request, requestLocal: RequestLocal): CompletableFuture[Unit] = { + def handleEnvelopeRequest(request: Request, requestLocal: RequestLocal): CompletableFuture[Unit] = { if (!authHelper.authorize(request.context, CLUSTER_ACTION, CLUSTER, CLUSTER_NAME)) { requestHelper.sendErrorResponseMaybeThrottle(request, new ClusterAuthorizationException( s"Principal ${request.context.principal} does not have required CLUSTER_ACTION for envelope")) @@ -171,13 +171,13 @@ class ControllerApis( CompletableFuture.completedFuture[Unit](()) } - def handleSaslHandshakeRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleSaslHandshakeRequest(request: Request): CompletableFuture[Unit] = { val responseData = new SaslHandshakeResponseData().setErrorCode(ILLEGAL_SASL_STATE.code) requestHelper.sendResponseMaybeThrottle(request, _ => new SaslHandshakeResponse(responseData)) CompletableFuture.completedFuture[Unit](()) } - def handleSaslAuthenticateRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleSaslAuthenticateRequest(request: Request): CompletableFuture[Unit] = { val responseData = new SaslAuthenticateResponseData() .setErrorCode(ILLEGAL_SASL_STATE.code) .setErrorMessage("SaslAuthenticate request received after successful authentication") @@ -185,18 +185,18 @@ class ControllerApis( CompletableFuture.completedFuture[Unit](()) } - def handleFetch(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleFetch(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) handleRaftRequest(request, response => FetchResponse.of(response.asInstanceOf[FetchResponseData])) } - def handleFetchSnapshot(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleFetchSnapshot(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) handleRaftRequest(request, response => new FetchSnapshotResponse(response.asInstanceOf[FetchSnapshotResponseData])) } - private def handleDeleteTopics(request: RequestChannel.Request): CompletableFuture[Unit] = { - val deleteTopicsRequest = request.body[DeleteTopicsRequest] + private def handleDeleteTopics(request: Request): CompletableFuture[Unit] = { + val deleteTopicsRequest = request.body(classOf[DeleteTopicsRequest]) val controllerMutationQuota = quotas.controllerMutation.newQuotaFor(request.session, request.header, 5) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, requestTimeoutMsToDeadlineNs(time, deleteTopicsRequest.data.timeoutMs), @@ -359,8 +359,8 @@ class ControllerApis( } } - private def handleCreateTopics(request: RequestChannel.Request): CompletableFuture[Unit] = { - val createTopicsRequest = request.body[CreateTopicsRequest] + private def handleCreateTopics(request: Request): CompletableFuture[Unit] = { + val createTopicsRequest = request.body(classOf[CreateTopicsRequest]) val controllerMutationQuota = quotas.controllerMutation.newQuotaFor(request.session, request.header, 6) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, requestTimeoutMsToDeadlineNs(time, createTopicsRequest.data.timeoutMs), @@ -448,14 +448,14 @@ class ControllerApis( } } - def handleApiVersionsRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleApiVersionsRequest(request: Request): CompletableFuture[Unit] = { // Note that controller returns its full list of supported ApiKeys and versions regardless of current // authentication state (e.g., before SASL authentication on an SASL listener, do note that no // Kafka protocol requests may take place on an SSL listener before the SSL handshake is finished). // If this is considered to leak information about the controller version a workaround is to use SSL // with client authentication which is performed at an earlier stage of the connection where the // ApiVersionRequest is not available. - val apiVersionRequest = request.body[ApiVersionsRequest] + val apiVersionRequest = request.body(classOf[ApiVersionsRequest]) if (apiVersionRequest.hasUnsupportedRequestVersion) { requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => apiVersionRequest.getErrorResponse(requestThrottleMs, UNSUPPORTED_VERSION.exception)) @@ -494,9 +494,9 @@ class ControllerApis( } } - def handleLegacyAlterConfigs(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleLegacyAlterConfigs(request: Request): CompletableFuture[Unit] = { val response = new AlterConfigsResponseData() - val alterConfigsRequest = request.body[AlterConfigsRequest] + val alterConfigsRequest = request.body(classOf[AlterConfigsRequest]) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) val duplicateResources = new util.HashSet[ConfigResource] val configChanges = new util.HashMap[ConfigResource, util.Map[String, String]]() @@ -553,29 +553,29 @@ class ControllerApis( } } - def handleVote(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleVote(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) handleRaftRequest(request, response => new VoteResponse(response.asInstanceOf[VoteResponseData])) } - def handleBeginQuorumEpoch(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleBeginQuorumEpoch(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) handleRaftRequest(request, response => new BeginQuorumEpochResponse(response.asInstanceOf[BeginQuorumEpochResponseData])) } - def handleEndQuorumEpoch(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleEndQuorumEpoch(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) handleRaftRequest(request, response => new EndQuorumEpochResponse(response.asInstanceOf[EndQuorumEpochResponseData])) } - def handleDescribeQuorum(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleDescribeQuorum(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, DESCRIBE) handleRaftRequest(request, response => new DescribeQuorumResponse(response.asInstanceOf[DescribeQuorumResponseData])) } - def handleElectLeaders(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleElectLeaders(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, ALTER) - val electLeadersRequest = request.body[ElectLeadersRequest] + val electLeadersRequest = request.body(classOf[ElectLeadersRequest]) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, requestTimeoutMsToDeadlineNs(time, electLeadersRequest.data.timeoutMs)) val future = controller.electLeaders(context, electLeadersRequest.data) @@ -592,8 +592,8 @@ class ControllerApis( } } - def handleAlterPartitionRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val alterPartitionRequest = request.body[AlterPartitionRequest] + def handleAlterPartitionRequest(request: Request): CompletableFuture[Unit] = { + val alterPartitionRequest = request.body(classOf[AlterPartitionRequest]) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) @@ -608,8 +608,8 @@ class ControllerApis( } } - def handleBrokerHeartBeatRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val heartbeatRequest = request.body[BrokerHeartbeatRequest] + def handleBrokerHeartBeatRequest(request: Request): CompletableFuture[Unit] = { + val heartbeatRequest = request.body(classOf[BrokerHeartbeatRequest]) authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, requestTimeoutMsToDeadlineNs(time, config.brokerHeartbeatIntervalMs / 2)) @@ -635,8 +635,8 @@ class ControllerApis( } } - def handleUnregisterBroker(request: RequestChannel.Request): CompletableFuture[Unit] = { - val decommissionRequest = request.body[UnregisterBrokerRequest] + def handleUnregisterBroker(request: Request): CompletableFuture[Unit] = { + val decommissionRequest = request.body(classOf[UnregisterBrokerRequest]) authHelper.authorizeClusterOperation(request, ALTER) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) @@ -656,8 +656,8 @@ class ControllerApis( } } - private def handleBrokerRegistration(request: RequestChannel.Request): CompletableFuture[Unit] = { - val registrationRequest = request.body[BrokerRegistrationRequest] + private def handleBrokerRegistration(request: Request): CompletableFuture[Unit] = { + val registrationRequest = request.body(classOf[BrokerRegistrationRequest]) authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) @@ -682,9 +682,9 @@ class ControllerApis( } } - private def handleRaftRequest(request: RequestChannel.Request, + private def handleRaftRequest(request: Request, buildResponse: ApiMessage => AbstractResponse): CompletableFuture[Unit] = { - val requestBody = request.body[AbstractRequest] + val requestBody = request.body(classOf[AbstractRequest]) val future = raftManager.handleRequest(request.context, request.header, requestBody.data, time.milliseconds()) future.handle[Unit] { (responseData, exception) => val response = if (exception != null) { @@ -696,8 +696,8 @@ class ControllerApis( } } - def handleAlterClientQuotas(request: RequestChannel.Request): CompletableFuture[Unit] = { - val quotaRequest = request.body[AlterClientQuotasRequest] + def handleAlterClientQuotas(request: Request): CompletableFuture[Unit] = { + val quotaRequest = request.body(classOf[AlterClientQuotasRequest]) authHelper.authorizeClusterOperation(request, ALTER_CONFIGS) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) @@ -712,9 +712,9 @@ class ControllerApis( } } - def handleIncrementalAlterConfigs(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleIncrementalAlterConfigs(request: Request): CompletableFuture[Unit] = { val response = new IncrementalAlterConfigsResponseData() - val alterConfigsRequest = request.body[IncrementalAlterConfigsRequest] + val alterConfigsRequest = request.body(classOf[IncrementalAlterConfigsRequest]) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) val duplicateResources = new util.HashSet[ConfigResource] @@ -793,11 +793,11 @@ class ControllerApis( } } - private def handleCreatePartitions(request: RequestChannel.Request): CompletableFuture[Unit] = { + private def handleCreatePartitions(request: Request): CompletableFuture[Unit] = { def filterAlterAuthorizedTopics(topics: Iterable[String]): Set[String] = { authHelper.filterByAuthorized(request.context, ALTER, TOPIC, topics)(n => n) } - val createPartitionsRequest = request.body[CreatePartitionsRequest] + val createPartitionsRequest = request.body(classOf[CreatePartitionsRequest]) val controllerMutationQuota = quotas.controllerMutation.newQuotaFor(request.session, request.header, 3) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, requestTimeoutMsToDeadlineNs(time, createPartitionsRequest.data.timeoutMs), @@ -816,7 +816,7 @@ class ControllerApis( } } - def handleDescribeConfigsRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleDescribeConfigsRequest(request: Request): CompletableFuture[Unit] = { val responseData = configHelper.handleDescribeConfigsRequest(request, authHelper) requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => new DescribeConfigsResponse(responseData.setThrottleTimeMs(requestThrottleMs))) @@ -861,8 +861,8 @@ class ControllerApis( } } - def handleControllerRegistration(request: RequestChannel.Request): CompletableFuture[Unit] = { - val registrationRequest = request.body[ControllerRegistrationRequest] + def handleControllerRegistration(request: Request): CompletableFuture[Unit] = { + val registrationRequest = request.body(classOf[ControllerRegistrationRequest]) authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) @@ -875,8 +875,8 @@ class ControllerApis( } } - def handleAlterPartitionReassignments(request: RequestChannel.Request): CompletableFuture[Unit] = { - val alterRequest = request.body[AlterPartitionReassignmentsRequest] + def handleAlterPartitionReassignments(request: Request): CompletableFuture[Unit] = { + val alterRequest = request.body(classOf[AlterPartitionReassignmentsRequest]) authHelper.authorizeClusterOperation(request, ALTER) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, requestTimeoutMsToDeadlineNs(time, alterRequest.data.timeoutMs)) @@ -887,8 +887,8 @@ class ControllerApis( } } - private def handleAlterUserScramCredentials(request: RequestChannel.Request): CompletableFuture[Unit] = { - val alterRequest = request.body[AlterUserScramCredentialsRequest] + private def handleAlterUserScramCredentials(request: Request): CompletableFuture[Unit] = { + val alterRequest = request.body(classOf[AlterUserScramCredentialsRequest]) authHelper.authorizeClusterOperation(request, ALTER) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) @@ -903,7 +903,7 @@ class ControllerApis( // The security protocol in the context is for the current connection (hop) // We need to always disallow a tokenAuthenticated principal // We need to allow special protocols but only in the forwarded case for testing. - def allowTokenRequests(request: RequestChannel.Request): Boolean = { + def allowTokenRequests(request: Request): Boolean = { val protocol = request.context.securityProtocol if (request.context.principal.tokenAuthenticated || // We allow forwarded requests to use PLAINTEXT for testing purposes @@ -915,8 +915,8 @@ class ControllerApis( true } - private def handleCreateDelegationTokenRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val createTokenRequest = request.body[CreateDelegationTokenRequest] + private def handleCreateDelegationTokenRequest(request: Request): CompletableFuture[Unit] = { + val createTokenRequest = request.body(classOf[CreateDelegationTokenRequest]) val requester = request.context.principal val ownerPrincipalName = createTokenRequest.data.ownerPrincipalName @@ -963,8 +963,8 @@ class ControllerApis( } } - private def handleRenewDelegationTokenRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val renewTokenRequest = request.body[RenewDelegationTokenRequest] + private def handleRenewDelegationTokenRequest(request: Request): CompletableFuture[Unit] = { + val renewTokenRequest = request.body(classOf[RenewDelegationTokenRequest]) if (!allowTokenRequests(request)) { requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => @@ -987,8 +987,8 @@ class ControllerApis( } } - private def handleExpireDelegationTokenRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val expireTokenRequest = request.body[ExpireDelegationTokenRequest] + private def handleExpireDelegationTokenRequest(request: Request): CompletableFuture[Unit] = { + val expireTokenRequest = request.body(classOf[ExpireDelegationTokenRequest]) if (!allowTokenRequests(request)) { requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => @@ -1011,8 +1011,8 @@ class ControllerApis( } } - def handleListPartitionReassignments(request: RequestChannel.Request): CompletableFuture[Unit] = { - val listRequest = request.body[ListPartitionReassignmentsRequest] + def handleListPartitionReassignments(request: Request): CompletableFuture[Unit] = { + val listRequest = request.body(classOf[ListPartitionReassignmentsRequest]) authHelper.authorizeClusterOperation(request, DESCRIBE) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) @@ -1023,8 +1023,8 @@ class ControllerApis( } } - def handleAllocateProducerIdsRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val allocatedProducerIdsRequest = request.body[AllocateProducerIdsRequest] + def handleAllocateProducerIdsRequest(request: Request): CompletableFuture[Unit] = { + val allocatedProducerIdsRequest = request.body(classOf[AllocateProducerIdsRequest]) authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) @@ -1041,8 +1041,8 @@ class ControllerApis( } } - def handleUpdateFeatures(request: RequestChannel.Request): CompletableFuture[Unit] = { - val updateFeaturesRequest = request.body[UpdateFeaturesRequest] + def handleUpdateFeatures(request: Request): CompletableFuture[Unit] = { + val updateFeaturesRequest = request.body(classOf[UpdateFeaturesRequest]) authHelper.authorizeClusterOperation(request, ALTER) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) @@ -1057,7 +1057,7 @@ class ControllerApis( } } - def handleDescribeCluster(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleDescribeCluster(request: Request): CompletableFuture[Unit] = { // Nearly all RPCs should check MetadataVersion inside the QuorumController. However, this // RPC is consulting a cache which lives outside the QC. So we check MetadataVersion here. if (!apiVersionManager.features.metadataVersion().isControllerRegistrationSupported) { @@ -1079,9 +1079,9 @@ class ControllerApis( CompletableFuture.completedFuture[Unit](()) } - private def handleAssignReplicasToDirs(request: RequestChannel.Request): CompletableFuture[Unit] = { + private def handleAssignReplicasToDirs(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) - val assignReplicasToDirsRequest = request.body[AssignReplicasToDirsRequest] + val assignReplicasToDirsRequest = request.body(classOf[AssignReplicasToDirsRequest]) val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty()) controller.assignReplicasToDirs(context, assignReplicasToDirsRequest.data).thenApply { reply => @@ -1090,17 +1090,17 @@ class ControllerApis( } } - def handleAddRaftVoter(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleAddRaftVoter(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, ALTER) handleRaftRequest(request, response => new AddRaftVoterResponse(response.asInstanceOf[AddRaftVoterResponseData])) } - def handleRemoveRaftVoter(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleRemoveRaftVoter(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, ALTER) handleRaftRequest(request, response => new RemoveRaftVoterResponse(response.asInstanceOf[RemoveRaftVoterResponseData])) } - def handleUpdateRaftVoter(request: RequestChannel.Request): CompletableFuture[Unit] = { + def handleUpdateRaftVoter(request: Request): CompletableFuture[Unit] = { authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) handleRaftRequest(request, response => new UpdateRaftVoterResponse(response.asInstanceOf[UpdateRaftVoterResponseData])) } diff --git a/core/src/main/scala/kafka/server/EnvelopeUtils.scala b/core/src/main/scala/kafka/server/EnvelopeUtils.scala index 3fdbf337033e3..0d364c7fcf642 100644 --- a/core/src/main/scala/kafka/server/EnvelopeUtils.scala +++ b/core/src/main/scala/kafka/server/EnvelopeUtils.scala @@ -19,23 +19,24 @@ package kafka.server import java.net.{InetAddress, UnknownHostException} import java.nio.ByteBuffer -import kafka.network.RequestChannel import org.apache.kafka.common.errors.{InvalidRequestException, PrincipalDeserializationException, UnsupportedVersionException} import org.apache.kafka.common.network.ClientInformation import org.apache.kafka.common.requests.{EnvelopeRequest, RequestContext, RequestHeader} import org.apache.kafka.common.security.auth.KafkaPrincipal +import org.apache.kafka.network.Request import org.apache.kafka.network.metrics.RequestChannelMetrics +import java.util.Optional import scala.jdk.OptionConverters.RichOptional object EnvelopeUtils { def handleEnvelopeRequest( - request: RequestChannel.Request, + request: Request, requestChannelMetrics: RequestChannelMetrics, - handler: RequestChannel.Request => Unit + handler: Request => Unit ): Unit = { - val envelope = request.body[EnvelopeRequest] + val envelope = request.body(classOf[EnvelopeRequest]) val forwardedPrincipal = parseForwardedPrincipal(request.context, envelope.requestPrincipal) val forwardedClientAddress = parseForwardedClientAddress(envelope.clientAddress) @@ -79,23 +80,23 @@ object EnvelopeUtils { } private def parseForwardedRequest( - envelope: RequestChannel.Request, + envelope: Request, forwardedContext: RequestContext, buffer: ByteBuffer, requestChannelMetrics: RequestChannelMetrics - ): RequestChannel.Request = { + ): Request = { try { - val forwardedRequest = new RequestChannel.Request( - processor = envelope.processor, - context = forwardedContext, - startTimeNanos = envelope.startTimeNanos, + val forwardedRequest = new Request( + envelope.processor, + forwardedContext, + envelope.startTimeNanos, envelope.memoryPool, buffer, requestChannelMetrics, - Some(envelope) + Optional.of(envelope) ) // set the dequeue time of forwardedRequest as the value of envelope request - forwardedRequest.requestDequeueTimeNanos = envelope.requestDequeueTimeNanos + forwardedRequest.requestDequeueTimeNanos(envelope.requestDequeueTimeNanos) forwardedRequest } catch { case e: InvalidRequestException => diff --git a/core/src/main/scala/kafka/server/ForwardingManager.scala b/core/src/main/scala/kafka/server/ForwardingManager.scala index 08a2d9d3ede6a..26e012550e063 100644 --- a/core/src/main/scala/kafka/server/ForwardingManager.scala +++ b/core/src/main/scala/kafka/server/ForwardingManager.scala @@ -18,13 +18,13 @@ package kafka.server import java.nio.ByteBuffer -import kafka.network.RequestChannel import kafka.utils.Logging import org.apache.kafka.clients.{ClientResponse, NodeApiVersions} import org.apache.kafka.common.errors.TimeoutException import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, EnvelopeResponse, RequestContext, RequestHeader} +import org.apache.kafka.network.Request import org.apache.kafka.server.ForwardingManagerUtil import org.apache.kafka.server.common.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import org.apache.kafka.server.metrics.ForwardingManagerMetrics @@ -36,7 +36,7 @@ trait ForwardingManager { def close(): Unit def forwardRequest( - originalRequest: RequestChannel.Request, + originalRequest: Request, responseCallback: Option[AbstractResponse] => Unit ): Unit = { val buffer = originalRequest.buffer.duplicate() @@ -44,13 +44,13 @@ trait ForwardingManager { forwardRequest(originalRequest.context, buffer, originalRequest.startTimeNanos, - originalRequest.body[AbstractRequest], + originalRequest.body(classOf[AbstractRequest]), () => originalRequest.toString, responseCallback) } def forwardRequest( - originalRequest: RequestChannel.Request, + originalRequest: Request, newRequestBody: AbstractRequest, responseCallback: Option[AbstractResponse] => Unit ): Unit = { diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index a214f3e7ed87d..6e68853dda4a6 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -61,6 +61,7 @@ import org.apache.kafka.coordinator.group.modern.share.ShareGroupConfigProvider import org.apache.kafka.coordinator.group.{Group, GroupConfig, GroupConfigManager, GroupCoordinator} import org.apache.kafka.coordinator.share.ShareCoordinator import org.apache.kafka.metadata.{ConfigRepository, MetadataCache} +import org.apache.kafka.network.Request import org.apache.kafka.security.DelegationTokenManager import org.apache.kafka.server.{ApiVersionManager, ClientMetricsManager, FetchManager, ProcessRole} import org.apache.kafka.server.authorizer._ @@ -129,7 +130,7 @@ class KafkaApis(val requestChannel: RequestChannel, info("Shutdown complete.") } - private def forwardToController(request: RequestChannel.Request): Unit = { + private def forwardToController(request: Request): Unit = { def responseCallback(responseOpt: Option[AbstractResponse]): Unit = { responseOpt match { case Some(response) => requestHelper.sendForwardedResponse(request, response) @@ -140,7 +141,7 @@ class KafkaApis(val requestChannel: RequestChannel, forwardingManager.forwardRequest(request, responseCallback) } - private def handleInvalidVersionsDuringForwarding(request: RequestChannel.Request): Unit = { + private def handleInvalidVersionsDuringForwarding(request: Request): Unit = { info(s"The client connection will be closed due to controller responded " + s"unsupported version exception during $request forwarding. " + s"This could happen when the controller changed after the connection was established.") @@ -150,7 +151,7 @@ class KafkaApis(val requestChannel: RequestChannel, /** * Top-level method that handles all requests and multiplexes to the right api */ - override def handle(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { + override def handle(request: Request, requestLocal: RequestLocal): Unit = { def handleError(e: Throwable): Unit = { error(s"Unexpected error handling request ${request.requestDesc(true)} " + s"with context ${request.context}", e) @@ -260,7 +261,7 @@ class KafkaApis(val requestChannel: RequestChannel, replicaManager.tryCompleteActions() // The local completion time may be set while processing the request. Only record it if it's unset. if (request.apiLocalCompleteTimeNanos < 0) - request.apiLocalCompleteTimeNanos = time.nanoseconds + request.apiLocalCompleteTimeNanos(time.nanoseconds) } } @@ -272,10 +273,10 @@ class KafkaApis(val requestChannel: RequestChannel, * Handle an offset commit request */ def handleOffsetCommitRequest( - request: RequestChannel.Request, + request: Request, requestLocal: RequestLocal ): CompletableFuture[Unit] = { - val offsetCommitRequest = request.body[OffsetCommitRequest] + val offsetCommitRequest = request.body(classOf[OffsetCommitRequest]) // Reject the request if not authorized to the group. if (!authHelper.authorize(request.context, READ, GROUP, offsetCommitRequest.data.groupId)) { @@ -391,8 +392,8 @@ class KafkaApis(val requestChannel: RequestChannel, /** * Handle a produce request */ - def handleProduceRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { - val produceRequest = request.body[ProduceRequest] + def handleProduceRequest(request: Request, requestLocal: RequestLocal): Unit = { + val produceRequest = request.body(classOf[ProduceRequest]) if (RequestUtils.hasTransactionalRecords(produceRequest)) { val isAuthorizedTransactional = produceRequest.transactionalId != null && @@ -493,7 +494,7 @@ class KafkaApis(val requestChannel: RequestChannel, else quotas.request.maybeRecordAndGetThrottleTimeMs(request, timeMs) val maxThrottleTimeMs = Math.max(bandwidthThrottleTimeMs, requestThrottleTimeMs) if (maxThrottleTimeMs > 0) { - request.apiThrottleTimeMs = maxThrottleTimeMs + request.apiThrottleTimeMs(maxThrottleTimeMs) if (bandwidthThrottleTimeMs > requestThrottleTimeMs) { requestHelper.throttle(quotas.produce, request, bandwidthThrottleTimeMs) } else { @@ -558,10 +559,10 @@ class KafkaApis(val requestChannel: RequestChannel, /** * Handle a fetch request */ - def handleFetchRequest(request: RequestChannel.Request): Unit = { + def handleFetchRequest(request: Request): Unit = { val versionId = request.header.apiVersion val clientId = request.header.clientId - val fetchRequest = request.body[FetchRequest] + val fetchRequest = request.body(classOf[FetchRequest]) val topicNames = if (fetchRequest.version() >= 13) metadataCache.topicIdsToNames() @@ -701,7 +702,7 @@ class KafkaApis(val requestChannel: RequestChannel, val maxThrottleTimeMs = math.max(bandwidthThrottleTimeMs, requestThrottleTimeMs) val fetchResponse = if (maxThrottleTimeMs > 0) { - request.apiThrottleTimeMs = maxThrottleTimeMs + request.apiThrottleTimeMs(maxThrottleTimeMs) // Even if we need to throttle for request quota violation, we should "unrecord" the already recorded value // from the fetch quota because we are going to return an empty response. quotas.fetch.unrecordQuotaSensor(request.session, request.header.clientId(), responseSize, timeMs) @@ -776,10 +777,10 @@ class KafkaApis(val requestChannel: RequestChannel, def replicationQuota(fetchRequest: FetchRequest): ReplicaQuota = if (fetchRequest.isFromFollower) quotas.leader else UNBOUNDED_QUOTA - def handleListOffsetRequest(request: RequestChannel.Request): Unit = { + def handleListOffsetRequest(request: Request): Unit = { val correlationId = request.header.correlationId val clientId = request.header.clientId - val offsetRequest = request.body[ListOffsetsRequest] + val offsetRequest = request.body(classOf[ListOffsetsRequest]) val version = request.header.apiVersion def buildErrorResponse(e: Errors, partition: ListOffsetsPartition): ListOffsetsPartitionResponse = { @@ -833,7 +834,7 @@ class KafkaApis(val requestChannel: RequestChannel, } private def getTopicMetadata( - request: RequestChannel.Request, + request: Request, fetchAllTopics: Boolean, allowAutoTopicCreation: Boolean, topics: Set[String], @@ -875,8 +876,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleTopicMetadataRequest(request: RequestChannel.Request): Unit = { - val metadataRequest = request.body[MetadataRequest] + def handleTopicMetadataRequest(request: Request): Unit = { + val metadataRequest = request.body(classOf[MetadataRequest]) val requestVersion = request.header.apiVersion // Topic IDs are not supported for versions 10 and 11. Topic names can not be null in these versions. @@ -999,7 +1000,7 @@ class KafkaApis(val requestChannel: RequestChannel, )) } - def handleDescribeTopicPartitionsRequest(request: RequestChannel.Request): Unit = { + def handleDescribeTopicPartitionsRequest(request: Request): Unit = { val response = describeTopicPartitionsRequestHandler.handleDescribeTopicPartitionsRequest(request) trace("Sending topic partitions metadata %s for correlation id %d to client %s".format(response.topics().asScala.mkString(","), request.header.correlationId, request.header.clientId)) @@ -1013,8 +1014,8 @@ class KafkaApis(val requestChannel: RequestChannel, /** * Handle an offset fetch request */ - def handleOffsetFetchRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val offsetFetchRequest = request.body[OffsetFetchRequest] + def handleOffsetFetchRequest(request: Request): CompletableFuture[Unit] = { + val offsetFetchRequest = request.body(classOf[OffsetFetchRequest]) val groups = offsetFetchRequest.groups() val requireStable = offsetFetchRequest.requireStable() @@ -1182,7 +1183,7 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleFindCoordinatorRequest(request: RequestChannel.Request): Unit = { + def handleFindCoordinatorRequest(request: Request): Unit = { val version = request.header.apiVersion if (version < 4) { handleFindCoordinatorRequestLessThanV4(request) @@ -1191,8 +1192,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - private def handleFindCoordinatorRequestV4AndAbove(request: RequestChannel.Request): Unit = { - val findCoordinatorRequest = request.body[FindCoordinatorRequest] + private def handleFindCoordinatorRequestV4AndAbove(request: Request): Unit = { + val findCoordinatorRequest = request.body(classOf[FindCoordinatorRequest]) val coordinators = findCoordinatorRequest.data.coordinatorKeys.asScala.map { key => val (error, node) = getCoordinator(request, findCoordinatorRequest.data.keyType, key) @@ -1215,8 +1216,8 @@ class KafkaApis(val requestChannel: RequestChannel, requestHelper.sendResponseMaybeThrottle(request, createResponse) } - private def handleFindCoordinatorRequestLessThanV4(request: RequestChannel.Request): Unit = { - val findCoordinatorRequest = request.body[FindCoordinatorRequest] + private def handleFindCoordinatorRequestLessThanV4(request: Request): Unit = { + val findCoordinatorRequest = request.body(classOf[FindCoordinatorRequest]) val (error, node) = getCoordinator(request, findCoordinatorRequest.data.keyType, findCoordinatorRequest.data.key) def createResponse(requestThrottleMs: Int): AbstractResponse = { @@ -1239,7 +1240,7 @@ class KafkaApis(val requestChannel: RequestChannel, } } - private def getCoordinator(request: RequestChannel.Request, keyType: Byte, key: String): (Errors, Node) = { + private def getCoordinator(request: Request, keyType: Byte, key: String): (Errors, Node) = { if (keyType == CoordinatorType.GROUP.id && !authHelper.authorize(request.context, DESCRIBE, GROUP, key)) (Errors.GROUP_AUTHORIZATION_FAILED, Node.noNode) @@ -1297,8 +1298,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleDescribeGroupsRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val describeRequest = request.body[DescribeGroupsRequest] + def handleDescribeGroupsRequest(request: Request): CompletableFuture[Unit] = { + val describeRequest = request.body(classOf[DescribeGroupsRequest]) val includeAuthorizedOperations = describeRequest.data.includeAuthorizedOperations val response = new DescribeGroupsResponseData() val authorizedGroups = new ArrayBuffer[String]() @@ -1345,8 +1346,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleListGroupsRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val listGroupsRequest = request.body[ListGroupsRequest] + def handleListGroupsRequest(request: Request): CompletableFuture[Unit] = { + val listGroupsRequest = request.body(classOf[ListGroupsRequest]) val hasClusterDescribe = authHelper.authorize(request.context, DESCRIBE, CLUSTER, CLUSTER_NAME, logIfDenied = false) groupCoordinator.listGroups( @@ -1372,10 +1373,10 @@ class KafkaApis(val requestChannel: RequestChannel, } def handleJoinGroupRequest( - request: RequestChannel.Request, + request: Request, requestLocal: RequestLocal ): CompletableFuture[Unit] = { - val joinGroupRequest = request.body[JoinGroupRequest] + val joinGroupRequest = request.body(classOf[JoinGroupRequest]) if (!authHelper.authorize(request.context, READ, GROUP, joinGroupRequest.data.groupId)) { requestHelper.sendMaybeThrottle(request, joinGroupRequest.getErrorResponse(Errors.GROUP_AUTHORIZATION_FAILED.exception)) @@ -1396,10 +1397,10 @@ class KafkaApis(val requestChannel: RequestChannel, } def handleSyncGroupRequest( - request: RequestChannel.Request, + request: Request, requestLocal: RequestLocal ): CompletableFuture[Unit] = { - val syncGroupRequest = request.body[SyncGroupRequest] + val syncGroupRequest = request.body(classOf[SyncGroupRequest]) if (!syncGroupRequest.areMandatoryProtocolTypeAndNamePresent()) { // Starting from version 5, ProtocolType and ProtocolName fields are mandatory. @@ -1424,10 +1425,10 @@ class KafkaApis(val requestChannel: RequestChannel, } def handleDeleteGroupsRequest( - request: RequestChannel.Request, + request: Request, requestLocal: RequestLocal ): CompletableFuture[Unit] = { - val deleteGroupsRequest = request.body[DeleteGroupsRequest] + val deleteGroupsRequest = request.body(classOf[DeleteGroupsRequest]) val groups = deleteGroupsRequest.data.groupsNames.asScala.distinct val (authorizedGroups, unauthorizedGroups) = @@ -1461,8 +1462,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleHeartbeatRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val heartbeatRequest = request.body[HeartbeatRequest] + def handleHeartbeatRequest(request: Request): CompletableFuture[Unit] = { + val heartbeatRequest = request.body(classOf[HeartbeatRequest]) if (!authHelper.authorize(request.context, READ, GROUP, heartbeatRequest.data.groupId)) { requestHelper.sendMaybeThrottle(request, heartbeatRequest.getErrorResponse(Errors.GROUP_AUTHORIZATION_FAILED.exception)) @@ -1481,8 +1482,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleLeaveGroupRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val leaveGroupRequest = request.body[LeaveGroupRequest] + def handleLeaveGroupRequest(request: Request): CompletableFuture[Unit] = { + val leaveGroupRequest = request.body(classOf[LeaveGroupRequest]) if (!authHelper.authorize(request.context, READ, GROUP, leaveGroupRequest.data.groupId)) { requestHelper.sendMaybeThrottle(request, leaveGroupRequest.getErrorResponse(Errors.GROUP_AUTHORIZATION_FAILED.exception)) @@ -1501,19 +1502,19 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleSaslHandshakeRequest(request: RequestChannel.Request): Unit = { + def handleSaslHandshakeRequest(request: Request): Unit = { val responseData = new SaslHandshakeResponseData().setErrorCode(Errors.ILLEGAL_SASL_STATE.code) requestHelper.sendResponseMaybeThrottle(request, _ => new SaslHandshakeResponse(responseData)) } - def handleSaslAuthenticateRequest(request: RequestChannel.Request): Unit = { + def handleSaslAuthenticateRequest(request: Request): Unit = { val responseData = new SaslAuthenticateResponseData() .setErrorCode(Errors.ILLEGAL_SASL_STATE.code) .setErrorMessage("SaslAuthenticate request received after successful authentication") requestHelper.sendResponseMaybeThrottle(request, _ => new SaslAuthenticateResponse(responseData)) } - def handleApiVersionsRequest(request: RequestChannel.Request): Unit = { + def handleApiVersionsRequest(request: Request): Unit = { // Note that broker returns its full list of supported ApiKeys and versions regardless of current // authentication state (e.g., before SASL authentication on an SASL listener, do note that no // Kafka protocol requests may take place on an SSL listener before the SSL handshake is finished). @@ -1521,7 +1522,7 @@ class KafkaApis(val requestChannel: RequestChannel, // with client authentication which is performed at an earlier stage of the connection where the // ApiVersionRequest is not available. def createResponseCallback(requestThrottleMs: Int): ApiVersionsResponse = { - val apiVersionRequest = request.body[ApiVersionsRequest] + val apiVersionRequest = request.body(classOf[ApiVersionsRequest]) if (apiVersionRequest.hasUnsupportedRequestVersion) { apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.UNSUPPORTED_VERSION.exception) } else if (!apiVersionRequest.isValid) { @@ -1533,8 +1534,8 @@ class KafkaApis(val requestChannel: RequestChannel, requestHelper.sendResponseMaybeThrottle(request, createResponseCallback) } - def handleDeleteRecordsRequest(request: RequestChannel.Request): Unit = { - val deleteRecordsRequest = request.body[DeleteRecordsRequest] + def handleDeleteRecordsRequest(request: Request): Unit = { + val deleteRecordsRequest = request.body(classOf[DeleteRecordsRequest]) val unauthorizedTopicResponses = mutable.Map[TopicPartition, DeleteRecordsPartitionResult]() val nonExistingTopicResponses = mutable.Map[TopicPartition, DeleteRecordsPartitionResult]() @@ -1598,8 +1599,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleInitProducerIdRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { - val initProducerIdRequest = request.body[InitProducerIdRequest] + def handleInitProducerIdRequest(request: Request, requestLocal: RequestLocal): Unit = { + val initProducerIdRequest = request.body(classOf[InitProducerIdRequest]) val transactionalId = initProducerIdRequest.data.transactionalId if (transactionalId != null) { @@ -1663,8 +1664,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleEndTxnRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { - val endTxnRequest = request.body[EndTxnRequest] + def handleEndTxnRequest(request: Request, requestLocal: RequestLocal): Unit = { + val endTxnRequest = request.body(classOf[EndTxnRequest]) val transactionalId = endTxnRequest.data.transactionalId if (authHelper.authorize(request.context, WRITE, TRANSACTIONAL_ID, transactionalId)) { @@ -1706,13 +1707,13 @@ class KafkaApis(val requestChannel: RequestChannel, ) } - def handleWriteTxnMarkersRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { + def handleWriteTxnMarkersRequest(request: Request, requestLocal: RequestLocal): Unit = { // We are checking for AlterCluster permissions first. If it is not present, we are authorizing cluster operation // The latter will throw an exception if it is denied. if (!authHelper.authorize(request.context, ALTER, CLUSTER, CLUSTER_NAME, logIfDenied = false)) { authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) } - val writeTxnMarkersRequest = request.body[WriteTxnMarkersRequest] + val writeTxnMarkersRequest = request.body(classOf[WriteTxnMarkersRequest]) val errors = new ConcurrentHashMap[java.lang.Long, util.Map[TopicPartition, Errors]]() // List of transaction marker entries, each containing producer metadata, transaction result (commit/abort), // target partitions, and transaction version. @@ -1851,12 +1852,12 @@ class KafkaApis(val requestChannel: RequestChannel, requestHelper.sendResponseExemptThrottle(request, new WriteTxnMarkersResponse(errors)) } - def handleAddPartitionsToTxnRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { + def handleAddPartitionsToTxnRequest(request: Request, requestLocal: RequestLocal): Unit = { val addPartitionsToTxnRequest = if (request.context.apiVersion() < 4) - request.body[AddPartitionsToTxnRequest].normalizeRequest() + request.body(classOf[AddPartitionsToTxnRequest]).normalizeRequest() else - request.body[AddPartitionsToTxnRequest] + request.body(classOf[AddPartitionsToTxnRequest]) val version = addPartitionsToTxnRequest.version val responses = new AddPartitionsToTxnResultCollection() val partitionsByTransaction = addPartitionsToTxnRequest.partitionsByTransaction() @@ -1963,8 +1964,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleAddOffsetsToTxnRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { - val addOffsetsToTxnRequest = request.body[AddOffsetsToTxnRequest] + def handleAddOffsetsToTxnRequest(request: Request, requestLocal: RequestLocal): Unit = { + val addOffsetsToTxnRequest = request.body(classOf[AddOffsetsToTxnRequest]) val transactionalId = addOffsetsToTxnRequest.data.transactionalId val groupId = addOffsetsToTxnRequest.data.groupId val offsetTopicPartition = new TopicPartition(GROUP_METADATA_TOPIC_NAME, groupCoordinator.partitionFor(groupId)) @@ -2014,10 +2015,10 @@ class KafkaApis(val requestChannel: RequestChannel, } def handleTxnOffsetCommitRequest( - request: RequestChannel.Request, + request: Request, requestLocal: RequestLocal ): CompletableFuture[Unit] = { - val txnOffsetCommitRequest = request.body[TxnOffsetCommitRequest] + val txnOffsetCommitRequest = request.body(classOf[TxnOffsetCommitRequest]) def sendResponse(response: TxnOffsetCommitResponse): Unit = { // We need to replace COORDINATOR_LOAD_IN_PROGRESS with COORDINATOR_NOT_AVAILABLE @@ -2142,12 +2143,12 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleDescribeAcls(request: RequestChannel.Request): Unit = { + def handleDescribeAcls(request: Request): Unit = { aclApis.handleDescribeAcls(request) } - def handleOffsetForLeaderEpochRequest(request: RequestChannel.Request): Unit = { - val offsetForLeaderEpoch = request.body[OffsetsForLeaderEpochRequest] + def handleOffsetForLeaderEpochRequest(request: Request): Unit = { + val offsetForLeaderEpoch = request.body(classOf[OffsetsForLeaderEpochRequest]) val topics = offsetForLeaderEpoch.data.topics.asScala.toSeq // The OffsetsForLeaderEpoch API was initially only used for inter-broker communication and required @@ -2181,8 +2182,8 @@ class KafkaApis(val requestChannel: RequestChannel, .setTopics(endOffsetsForAllTopics))) } - def handleAlterConfigsRequest(request: RequestChannel.Request): Unit = { - val original = request.body[AlterConfigsRequest] + def handleAlterConfigsRequest(request: Request): Unit = { + val original = request.body(classOf[AlterConfigsRequest]) val preprocessingResponses = configManager.preprocess(original.data()) val remaining = ConfigAdminManager.copyWithoutPreprocessed(original.data(), preprocessingResponses) def sendResponse(secondPart: Option[ApiMessage]): Unit = { @@ -2205,8 +2206,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleIncrementalAlterConfigsRequest(request: RequestChannel.Request): Unit = { - val original = request.body[IncrementalAlterConfigsRequest] + def handleIncrementalAlterConfigsRequest(request: Request): Unit = { + val original = request.body(classOf[IncrementalAlterConfigsRequest]) val preprocessingResponses = configManager.preprocess(original.data(), (rType, rName) => authHelper.authorize(request.context, ALTER_CONFIGS, rType, rName)) val remaining = ConfigAdminManager.copyWithoutPreprocessed(original.data(), preprocessingResponses) @@ -2232,14 +2233,14 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleDescribeConfigsRequest(request: RequestChannel.Request): Unit = { + def handleDescribeConfigsRequest(request: Request): Unit = { val responseData = configHelper.handleDescribeConfigsRequest(request, authHelper) requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => new DescribeConfigsResponse(responseData.setThrottleTimeMs(requestThrottleMs))) } - def handleAlterReplicaLogDirsRequest(request: RequestChannel.Request): Unit = { - val alterReplicaDirsRequest = request.body[AlterReplicaLogDirsRequest] + def handleAlterReplicaLogDirsRequest(request: Request): Unit = { + val alterReplicaDirsRequest = request.body(classOf[AlterReplicaLogDirsRequest]) if (authHelper.authorize(request.context, ALTER, CLUSTER, CLUSTER_NAME)) { val result = replicaManager.alterReplicaLogDirs(alterReplicaDirsRequest.partitionDirs.asScala) requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => @@ -2260,8 +2261,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleDescribeLogDirsRequest(request: RequestChannel.Request): Unit = { - val describeLogDirsDirRequest = request.body[DescribeLogDirsRequest] + def handleDescribeLogDirsRequest(request: Request): Unit = { + val describeLogDirsDirRequest = request.body(classOf[DescribeLogDirsRequest]) val (logDirInfos, error) = { if (authHelper.authorize(request.context, DESCRIBE, CLUSTER, CLUSTER_NAME)) { val partitions = @@ -2283,8 +2284,8 @@ class KafkaApis(val requestChannel: RequestChannel, .setErrorCode(error.code))) } - def handleCreateTokenRequest(request: RequestChannel.Request): Unit = { - val createTokenRequest = request.body[CreateDelegationTokenRequest] + def handleCreateTokenRequest(request: Request): Unit = { + val createTokenRequest = request.body(classOf[CreateDelegationTokenRequest]) val requester = request.context.principal val ownerPrincipalName = createTokenRequest.data.ownerPrincipalName @@ -2313,7 +2314,7 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleExpireTokenRequest(request: RequestChannel.Request): Unit = { + def handleExpireTokenRequest(request: Request): Unit = { if (!allowTokenRequests(request)) { requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => new ExpireDelegationTokenResponse( @@ -2326,7 +2327,7 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleRenewTokenRequest(request: RequestChannel.Request): Unit = { + def handleRenewTokenRequest(request: Request): Unit = { if (!allowTokenRequests(request)) { requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => new RenewDelegationTokenResponse( @@ -2339,8 +2340,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleDescribeTokensRequest(request: RequestChannel.Request): Unit = { - val describeTokenRequest = request.body[DescribeDelegationTokenRequest] + def handleDescribeTokensRequest(request: Request): Unit = { + val describeTokenRequest = request.body(classOf[DescribeDelegationTokenRequest]) // the callback for sending a describe token response def sendResponseCallback(error: Errors, tokenDetails: util.List[DelegationToken]): Unit = { @@ -2375,7 +2376,7 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def allowTokenRequests(request: RequestChannel.Request): Boolean = { + def allowTokenRequests(request: Request): Boolean = { val protocol = request.context.securityProtocol if (request.context.principal.tokenAuthenticated || protocol == SecurityProtocol.PLAINTEXT || @@ -2387,10 +2388,10 @@ class KafkaApis(val requestChannel: RequestChannel, } def handleOffsetDeleteRequest( - request: RequestChannel.Request, + request: Request, requestLocal: RequestLocal ): CompletableFuture[Unit] = { - val offsetDeleteRequest = request.body[OffsetDeleteRequest] + val offsetDeleteRequest = request.body(classOf[OffsetDeleteRequest]) if (!authHelper.authorize(request.context, DELETE, GROUP, offsetDeleteRequest.data.groupId)) { requestHelper.sendMaybeThrottle(request, offsetDeleteRequest.getErrorResponse(Errors.GROUP_AUTHORIZATION_FAILED.exception)) @@ -2450,8 +2451,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleDescribeClientQuotasRequest(request: RequestChannel.Request): Unit = { - val describeClientQuotasRequest = request.body[DescribeClientQuotasRequest] + def handleDescribeClientQuotasRequest(request: Request): Unit = { + val describeClientQuotasRequest = request.body(classOf[DescribeClientQuotasRequest]) if (!authHelper.authorize(request.context, DESCRIBE_CONFIGS, CLUSTER, CLUSTER_NAME)) { requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => @@ -2465,8 +2466,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleDescribeUserScramCredentialsRequest(request: RequestChannel.Request): Unit = { - val describeUserScramCredentialsRequest = request.body[DescribeUserScramCredentialsRequest] + def handleDescribeUserScramCredentialsRequest(request: Request): Unit = { + val describeUserScramCredentialsRequest = request.body(classOf[DescribeUserScramCredentialsRequest]) if (!authHelper.authorize(request.context, DESCRIBE, CLUSTER, CLUSTER_NAME)) { requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => @@ -2478,14 +2479,14 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleDescribeCluster(request: RequestChannel.Request): Unit = { + def handleDescribeCluster(request: Request): Unit = { val response = authHelper.computeDescribeClusterResponse( request, EndpointType.BROKER, clusterId, () => { val brokers = new DescribeClusterResponseData.DescribeClusterBrokerCollection() - val describeClusterRequest = request.body[DescribeClusterRequest] + val describeClusterRequest = request.body(classOf[DescribeClusterRequest]) metadataCache.getBrokerNodes(request.context.listenerName).forEach { node => if (!node.isFenced || describeClusterRequest.data().includeFencedBrokers()) { brokers.add(new DescribeClusterResponseData.DescribeClusterBroker(). @@ -2506,8 +2507,8 @@ class KafkaApis(val requestChannel: RequestChannel, new DescribeClusterResponse(response.setThrottleTimeMs(requestThrottleMs))) } - def handleDescribeProducersRequest(request: RequestChannel.Request): Unit = { - val describeProducersRequest = request.body[DescribeProducersRequest] + def handleDescribeProducersRequest(request: Request): Unit = { + val describeProducersRequest = request.body(classOf[DescribeProducersRequest]) def partitionError( topicPartition: TopicPartition, @@ -2561,8 +2562,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleDescribeTransactionsRequest(request: RequestChannel.Request): Unit = { - val describeTransactionsRequest = request.body[DescribeTransactionsRequest] + def handleDescribeTransactionsRequest(request: Request): Unit = { + val describeTransactionsRequest = request.body(classOf[DescribeTransactionsRequest]) val response = new DescribeTransactionsResponseData() describeTransactionsRequest.data.transactionalIds.forEach { transactionalId => @@ -2589,8 +2590,8 @@ class KafkaApis(val requestChannel: RequestChannel, new DescribeTransactionsResponse(response.setThrottleTimeMs(requestThrottleMs))) } - def handleListTransactionsRequest(request: RequestChannel.Request): Unit = { - val listTransactionsRequest = request.body[ListTransactionsRequest] + def handleListTransactionsRequest(request: Request): Unit = { + val listTransactionsRequest = request.body(classOf[ListTransactionsRequest]) val filteredProducerIds = listTransactionsRequest.data.producerIdFilters.asScala.map(Long.unbox).toSet val filteredStates = listTransactionsRequest.data.stateFilters.asScala.toSet val durationFilter = listTransactionsRequest.data.durationFilter() @@ -2625,8 +2626,8 @@ class KafkaApis(val requestChannel: RequestChannel, groupVersion().isConsumerRebalanceProtocolSupported } - def handleConsumerGroupHeartbeat(request: RequestChannel.Request): CompletableFuture[Unit] = { - val consumerGroupHeartbeatRequest = request.body[ConsumerGroupHeartbeatRequest] + def handleConsumerGroupHeartbeat(request: Request): CompletableFuture[Unit] = { + val consumerGroupHeartbeatRequest = request.body(classOf[ConsumerGroupHeartbeatRequest]) if (!isConsumerGroupProtocolEnabled()) { // The API is not supported by the "old" group coordinator (the default). If the @@ -2665,14 +2666,14 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleConsumerGroupDescribe(request: RequestChannel.Request): CompletableFuture[Unit] = { - val consumerGroupDescribeRequest = request.body[ConsumerGroupDescribeRequest] + def handleConsumerGroupDescribe(request: Request): CompletableFuture[Unit] = { + val consumerGroupDescribeRequest = request.body(classOf[ConsumerGroupDescribeRequest]) val includeAuthorizedOperations = consumerGroupDescribeRequest.data.includeAuthorizedOperations if (!isConsumerGroupProtocolEnabled()) { // The API is not supported by the "old" group coordinator (the default). If the // new one is not enabled, we fail directly here. - requestHelper.sendMaybeThrottle(request, request.body[ConsumerGroupDescribeRequest].getErrorResponse(Errors.UNSUPPORTED_VERSION.exception)) + requestHelper.sendMaybeThrottle(request, request.body(classOf[ConsumerGroupDescribeRequest]).getErrorResponse(Errors.UNSUPPORTED_VERSION.exception)) CompletableFuture.completedFuture[Unit](()) } else { val response = new ConsumerGroupDescribeResponseData() @@ -2760,8 +2761,8 @@ class KafkaApis(val requestChannel: RequestChannel, streamsVersion().streamsGroupSupported } - def handleStreamsGroupHeartbeat(request: RequestChannel.Request): CompletableFuture[Unit] = { - val streamsGroupHeartbeatRequest = request.body[StreamsGroupHeartbeatRequest] + def handleStreamsGroupHeartbeat(request: Request): CompletableFuture[Unit] = { + val streamsGroupHeartbeatRequest = request.body(classOf[StreamsGroupHeartbeatRequest]) if (!isStreamsGroupProtocolEnabled) { // The API is not enabled by default. If it is not enabled, we fail directly here. @@ -2891,13 +2892,13 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleStreamsGroupDescribe(request: RequestChannel.Request): CompletableFuture[Unit] = { - val streamsGroupDescribeRequest = request.body[StreamsGroupDescribeRequest] + def handleStreamsGroupDescribe(request: Request): CompletableFuture[Unit] = { + val streamsGroupDescribeRequest = request.body(classOf[StreamsGroupDescribeRequest]) val includeAuthorizedOperations = streamsGroupDescribeRequest.data.includeAuthorizedOperations if (!isStreamsGroupProtocolEnabled) { // The API is not enabled by default. If it is not enabled, we fail directly here. - requestHelper.sendMaybeThrottle(request, request.body[StreamsGroupDescribeRequest].getErrorResponse(Errors.UNSUPPORTED_VERSION.exception)) + requestHelper.sendMaybeThrottle(request, request.body(classOf[StreamsGroupDescribeRequest]).getErrorResponse(Errors.UNSUPPORTED_VERSION.exception)) CompletableFuture.completedFuture[Unit](()) } else { val response = new StreamsGroupDescribeResponseData() @@ -2991,8 +2992,8 @@ class KafkaApis(val requestChannel: RequestChannel, } - def handleGetTelemetrySubscriptionsRequest(request: RequestChannel.Request): Unit = { - val subscriptionRequest = request.body[GetTelemetrySubscriptionsRequest] + def handleGetTelemetrySubscriptionsRequest(request: Request): Unit = { + val subscriptionRequest = request.body(classOf[GetTelemetrySubscriptionsRequest]) try { requestHelper.sendMaybeThrottle(request, clientMetricsManager.processGetTelemetrySubscriptionRequest(subscriptionRequest, request.context)) } catch { @@ -3001,8 +3002,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - private def handlePushTelemetryRequest(request: RequestChannel.Request): Unit = { - val pushTelemetryRequest = request.body[PushTelemetryRequest] + private def handlePushTelemetryRequest(request: Request): Unit = { + val pushTelemetryRequest = request.body(classOf[PushTelemetryRequest]) try { requestHelper.sendMaybeThrottle(request, clientMetricsManager.processPushTelemetryRequest(pushTelemetryRequest, request.context)) } catch { @@ -3016,8 +3017,8 @@ class KafkaApis(val requestChannel: RequestChannel, * to retrieve config resources. If resourceTypes are specified, it returns matched config resources. * If a config resource type is not supported, the handler returns UNSUPPORTED_VERSION. */ - private def handleListConfigResources(request: RequestChannel.Request): Unit = { - val listConfigResourcesRequest = request.body[ListConfigResourcesRequest] + private def handleListConfigResources(request: Request): Unit = { + val listConfigResourcesRequest = request.body(classOf[ListConfigResourcesRequest]) if (!authHelper.authorize(request.context, DESCRIBE_CONFIGS, CLUSTER, CLUSTER_NAME)) { requestHelper.sendMaybeThrottle(request, listConfigResourcesRequest.getErrorResponse(Errors.CLUSTER_AUTHORIZATION_FAILED.exception)) @@ -3068,8 +3069,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleShareGroupHeartbeat(request: RequestChannel.Request): CompletableFuture[Unit] = { - val shareGroupHeartbeatRequest = request.body[ShareGroupHeartbeatRequest] + def handleShareGroupHeartbeat(request: Request): CompletableFuture[Unit] = { + val shareGroupHeartbeatRequest = request.body(classOf[ShareGroupHeartbeatRequest]) if (!isShareGroupProtocolEnabled) { requestHelper.sendMaybeThrottle(request, shareGroupHeartbeatRequest.getErrorResponse(Errors.UNSUPPORTED_VERSION.exception)) @@ -3111,8 +3112,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleShareGroupDescribe(request: RequestChannel.Request): CompletableFuture[Unit] = { - val shareGroupDescribeRequest = request.body[ShareGroupDescribeRequest] + def handleShareGroupDescribe(request: Request): CompletableFuture[Unit] = { + val shareGroupDescribeRequest = request.body(classOf[ShareGroupDescribeRequest]) val includeAuthorizedOperations = shareGroupDescribeRequest.data.includeAuthorizedOperations if (!isShareGroupProtocolEnabled) { @@ -3196,8 +3197,8 @@ class KafkaApis(val requestChannel: RequestChannel, /** * Handle a shareFetch request */ - def handleShareFetchRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val shareFetchRequest = request.body[ShareFetchRequest] + def handleShareFetchRequest(request: Request): CompletableFuture[Unit] = { + val shareFetchRequest = request.body(classOf[ShareFetchRequest]) if (!isShareGroupProtocolEnabled) { requestHelper.sendMaybeThrottle(request, shareFetchRequest.getErrorResponse(AbstractResponse.DEFAULT_THROTTLE_TIME, Errors.UNSUPPORTED_VERSION.exception)) @@ -3433,7 +3434,7 @@ class KafkaApis(val requestChannel: RequestChannel, } // Visible for Testing - def handleFetchFromShareFetchRequest(request: RequestChannel.Request, + def handleFetchFromShareFetchRequest(request: Request, shareSessionEpoch: Int, erroneousAndValidPartitionData: ErroneousAndValidPartitionData, sharePartitionManagerInstance: SharePartitionManager, @@ -3454,7 +3455,7 @@ class KafkaApis(val requestChannel: RequestChannel, interestedTopicPartitions.add(topicIdPartition) } - val shareFetchRequest = request.body[ShareFetchRequest] + val shareFetchRequest = request.body(classOf[ShareFetchRequest]) val clientId = request.header.clientId val groupId = shareFetchRequest.data.groupId @@ -3568,8 +3569,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleShareAcknowledgeRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val shareAcknowledgeRequest = request.body[ShareAcknowledgeRequest] + def handleShareAcknowledgeRequest(request: Request): CompletableFuture[Unit] = { + val shareAcknowledgeRequest = request.body(classOf[ShareAcknowledgeRequest]) if (!isShareGroupProtocolEnabled) { requestHelper.sendMaybeThrottle(request, @@ -3653,8 +3654,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleInitializeShareGroupStateRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val initializeShareGroupStateRequest = request.body[InitializeShareGroupStateRequest] + def handleInitializeShareGroupStateRequest(request: Request): CompletableFuture[Unit] = { + val initializeShareGroupStateRequest = request.body(classOf[InitializeShareGroupStateRequest]) // We do not need a check for isShareGroupProtocolEnabled in this RPC since there is a check for it in ShareFetch/ShareAcknowledge RPCs, // hence requests won't reach Persister. @@ -3677,8 +3678,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleReadShareGroupStateRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val readShareGroupStateRequest = request.body[ReadShareGroupStateRequest] + def handleReadShareGroupStateRequest(request: Request): CompletableFuture[Unit] = { + val readShareGroupStateRequest = request.body(classOf[ReadShareGroupStateRequest]) // We do not need a check for isShareGroupProtocolEnabled in this RPC since there is a check for it in ShareFetch/ShareAcknowledge RPCs, // hence requests won't reach Persister. @@ -3701,8 +3702,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleWriteShareGroupStateRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val writeShareGroupStateRequest = request.body[WriteShareGroupStateRequest] + def handleWriteShareGroupStateRequest(request: Request): CompletableFuture[Unit] = { + val writeShareGroupStateRequest = request.body(classOf[WriteShareGroupStateRequest]) // We do not need a check for isShareGroupProtocolEnabled in this RPC since there is a check for it in ShareFetch/ShareAcknowledge RPCs, // hence requests won't reach Persister. @@ -3725,8 +3726,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleDeleteShareGroupStateRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val deleteShareGroupStateRequest = request.body[DeleteShareGroupStateRequest] + def handleDeleteShareGroupStateRequest(request: Request): CompletableFuture[Unit] = { + val deleteShareGroupStateRequest = request.body(classOf[DeleteShareGroupStateRequest]) // We do not need a check for isShareGroupProtocolEnabled in this RPC since there is a check for it in ShareFetch/ShareAcknowledge RPCs, // hence requests won't reach Persister. @@ -3749,8 +3750,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleReadShareGroupStateSummaryRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val readShareGroupStateSummaryRequest = request.body[ReadShareGroupStateSummaryRequest] + def handleReadShareGroupStateSummaryRequest(request: Request): CompletableFuture[Unit] = { + val readShareGroupStateSummaryRequest = request.body(classOf[ReadShareGroupStateSummaryRequest]) // We do not need a check for isShareGroupProtocolEnabled in this RPC since there is a check for it in ShareFetch/ShareAcknowledge RPCs, // hence requests won't reach Persister. @@ -3773,8 +3774,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleDescribeShareGroupOffsetsRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val describeShareGroupOffsetsRequest = request.body[DescribeShareGroupOffsetsRequest] + def handleDescribeShareGroupOffsetsRequest(request: Request): CompletableFuture[Unit] = { + val describeShareGroupOffsetsRequest = request.body(classOf[DescribeShareGroupOffsetsRequest]) val groups = describeShareGroupOffsetsRequest.groups() val futures = new mutable.ArrayBuffer[CompletableFuture[DescribeShareGroupOffsetsResponseData.DescribeShareGroupOffsetsResponseGroup]](groups.size) @@ -3887,8 +3888,8 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleAlterShareGroupOffsetsRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val alterShareGroupOffsetsRequest = request.body[AlterShareGroupOffsetsRequest] + def handleAlterShareGroupOffsetsRequest(request: Request): CompletableFuture[Unit] = { + val alterShareGroupOffsetsRequest = request.body(classOf[AlterShareGroupOffsetsRequest]) val groupId = alterShareGroupOffsetsRequest.data.groupId if (!isShareGroupProtocolEnabled) { @@ -3938,8 +3939,8 @@ class KafkaApis(val requestChannel: RequestChannel, CompletableFuture.completedFuture[Unit](()) } - def handleDeleteShareGroupOffsetsRequest(request: RequestChannel.Request): CompletableFuture[Unit] = { - val deleteShareGroupOffsetsRequest = request.body[DeleteShareGroupOffsetsRequest] + def handleDeleteShareGroupOffsetsRequest(request: Request): CompletableFuture[Unit] = { + val deleteShareGroupOffsetsRequest = request.body(classOf[DeleteShareGroupOffsetsRequest]) val groupId = deleteShareGroupOffsetsRequest.data.groupId @@ -4069,8 +4070,8 @@ class KafkaApis(val requestChannel: RequestChannel, // the callback for processing a share acknowledge response, invoked before throttling def processShareAcknowledgeResponse(responseAcknowledgeData: Map[TopicIdPartition, ShareAcknowledgeResponseData.PartitionData], - request: RequestChannel.Request): ShareAcknowledgeResponse = { - val shareAcknowledgeRequest = request.body[ShareAcknowledgeRequest] + request: Request): ShareAcknowledgeResponse = { + val shareAcknowledgeRequest = request.body(classOf[ShareAcknowledgeRequest]) val groupId = shareAcknowledgeRequest.data.groupId val partitions = new util.LinkedHashMap[TopicIdPartition, ShareAcknowledgeResponseData.PartitionData] @@ -4149,12 +4150,12 @@ class KafkaApis(val requestChannel: RequestChannel, // the callback for processing a share fetch response. private def processShareFetchResponse(responsePartitionData: Map[TopicIdPartition, ShareFetchResponseData.PartitionData], - request: RequestChannel.Request, + request: Request, topicIdNames: util.Map[Uuid, String], shareFetchContext: ShareFetchContext): ShareFetchResponse = { val clientId = request.header.clientId val versionId = request.header.apiVersion - val shareFetchRequest = request.body[ShareFetchRequest] + val shareFetchRequest = request.body(classOf[ShareFetchRequest]) val groupId = shareFetchRequest.data.groupId val memberId = shareFetchRequest.data.memberId @@ -4223,7 +4224,7 @@ class KafkaApis(val requestChannel: RequestChannel, val maxThrottleTimeMs = math.max(bandwidthThrottleTimeMs, requestThrottleTimeMs) if (maxThrottleTimeMs > 0) { - request.apiThrottleTimeMs = maxThrottleTimeMs + request.apiThrottleTimeMs(maxThrottleTimeMs) // Even if we need to throttle for request quota violation, we should "unrecord" the already recorded value // from the fetch quota because we are going to return an empty response. quotas.fetch.unrecordQuotaSensor(request.session, request.header.clientId(), responseSize, timeMs) @@ -4274,7 +4275,7 @@ class KafkaApis(val requestChannel: RequestChannel, memberId != null && memberId.nonEmpty && memberId.length <= 36 } - private def updateRecordConversionStats(request: RequestChannel.Request, + private def updateRecordConversionStats(request: Request, tp: TopicPartition, conversionStats: RecordValidationStats): Unit = { val conversionCount = conversionStats.numRecordsConverted @@ -4289,12 +4290,12 @@ class KafkaApis(val requestChannel: RequestChannel, case _ => throw new IllegalStateException("Message conversion info is recorded only for Produce/Fetch requests") } - request.messageConversionsTimeNanos += conversionStats.conversionTimeNanos + request.messageConversionsTimeNanos(request.messageConversionsTimeNanos + conversionStats.conversionTimeNanos) } - request.temporaryMemoryBytes += conversionStats.temporaryMemoryBytes + request.temporaryMemoryBytes(request.temporaryMemoryBytes + conversionStats.temporaryMemoryBytes) } - def authorizeClusterOperation(request: RequestChannel.Request, operation: AclOperation): Boolean = { + def authorizeClusterOperation(request: Request, operation: AclOperation): Boolean = { authHelper.authorize(request.context, operation, CLUSTER, CLUSTER_NAME) } } diff --git a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala index 925364b0f1c50..c36d96e8380a5 100755 --- a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala +++ b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala @@ -27,20 +27,22 @@ import com.yammer.metrics.core.Meter import org.apache.kafka.common.internals.FatalExitError import org.apache.kafka.common.utils.internals.{Exit, KafkaThread} import org.apache.kafka.common.utils.Time +import org.apache.kafka.network.{CallbackRequest, Request, ShutdownRequest, WakeupRequest} import org.apache.kafka.server.common.RequestLocal import org.apache.kafka.server.metrics.KafkaMetricsGroup +import java.util.OptionalLong import scala.collection.mutable trait ApiRequestHandler { - def handle(request: RequestChannel.Request, requestLocal: RequestLocal): Unit + def handle(request: Request, requestLocal: RequestLocal): Unit def tryCompleteActions(): Unit = {} } object KafkaRequestHandler { // Support for scheduling callbacks on a request thread. private val threadRequestChannel = new ThreadLocal[RequestChannel] - private val threadCurrentRequest = new ThreadLocal[RequestChannel.Request] + private val threadCurrentRequest = new ThreadLocal[Request] // For testing @volatile private var bypassThreadCheck = false @@ -76,7 +78,7 @@ object KafkaRequestHandler { } else { // The requestChannel and request are captured in this lambda, so when it's executed on the callback thread // we can re-schedule the original callback on a request thread and update the metrics accordingly. - requestChannel.sendCallbackRequest(RequestChannel.CallbackRequest(newRequestLocal => asyncCompletionCallback(newRequestLocal, t), currentRequest)) + requestChannel.sendCallbackRequest(new CallbackRequest(newRequestLocal => asyncCompletionCallback(newRequestLocal, t), currentRequest)) } } } @@ -121,28 +123,29 @@ class KafkaRequestHandler( aggregateIdleMeter.mark(idleTime / aggregateThreads.get) req match { - case RequestChannel.ShutdownRequest => + case _: ShutdownRequest => debug(s"Kafka request handler $id on broker $brokerId received shut down command") completeShutdown() return - case callback: RequestChannel.CallbackRequest => + case callback: CallbackRequest => val originalRequest = callback.originalRequest try { - // If we've already executed a callback for this request, reset the times and subtract the callback time from the + // If we've already executed a callback for this request, reset the times and subtract the callback time from the // new dequeue time. This will allow calculation of multiple callback times. // Otherwise, set dequeue time to now. - if (originalRequest.callbackRequestDequeueTimeNanos.isDefined) { - val prevCallbacksTimeNanos = originalRequest.callbackRequestCompleteTimeNanos.getOrElse(0L) - originalRequest.callbackRequestDequeueTimeNanos.getOrElse(0L) - originalRequest.callbackRequestCompleteTimeNanos = None - originalRequest.callbackRequestDequeueTimeNanos = Some(time.nanoseconds() - prevCallbacksTimeNanos) + if (originalRequest.callbackRequestDequeueTimeNanos.isPresent) { + val prevCallbacksTimeNanos = originalRequest.callbackRequestCompleteTimeNanos.orElse(0L) - + originalRequest.callbackRequestDequeueTimeNanos.orElse(0L) + originalRequest.callbackRequestCompleteTimeNanos(OptionalLong.empty) + originalRequest.callbackRequestDequeueTimeNanos(OptionalLong.of(time.nanoseconds() - prevCallbacksTimeNanos)) } else { - originalRequest.callbackRequestDequeueTimeNanos = Some(time.nanoseconds()) + originalRequest.callbackRequestDequeueTimeNanos(OptionalLong.of(time.nanoseconds())) } - + threadCurrentRequest.set(originalRequest) - callback.fun(requestLocal) + callback.fun().accept(requestLocal) } catch { case e: FatalExitError => completeShutdown() @@ -152,13 +155,13 @@ class KafkaRequestHandler( // When handling requests, we try to complete actions after, so we should try to do so here as well. apis.tryCompleteActions() if (originalRequest.callbackRequestCompleteTimeNanos.isEmpty) - originalRequest.callbackRequestCompleteTimeNanos = Some(time.nanoseconds()) + originalRequest.callbackRequestCompleteTimeNanos(OptionalLong.of(time.nanoseconds())) threadCurrentRequest.remove() } - case request: RequestChannel.Request => + case request: Request => try { - request.requestDequeueTimeNanos = endTime + request.requestDequeueTimeNanos(endTime) trace(s"Kafka request handler $id on broker $brokerId handling request $request") threadCurrentRequest.set(request) apis.handle(request, requestLocal) @@ -172,7 +175,7 @@ class KafkaRequestHandler( request.releaseBuffer() } - case RequestChannel.WakeupRequest => + case _: WakeupRequest => // We should handle this in receiveRequest by polling callbackQueue. warn("Received a wakeup request outside of typical usage.") diff --git a/core/src/main/scala/kafka/server/RequestHandlerHelper.scala b/core/src/main/scala/kafka/server/RequestHandlerHelper.scala index 083e0d7cafff8..39a9cdf5f4fc2 100644 --- a/core/src/main/scala/kafka/server/RequestHandlerHelper.scala +++ b/core/src/main/scala/kafka/server/RequestHandlerHelper.scala @@ -23,6 +23,7 @@ import org.apache.kafka.common.errors.ClusterAuthorizationException import org.apache.kafka.common.network.Send import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse} import org.apache.kafka.common.utils.Time +import org.apache.kafka.network.Request import org.apache.kafka.server.quota.{ClientQuotaManager, ControllerMutationQuota, ThrottleCallback} class RequestHandlerHelper( @@ -33,7 +34,7 @@ class RequestHandlerHelper( def throttle( quotaManager: ClientQuotaManager, - request: RequestChannel.Request, + request: Request, throttleTimeMs: Int ): Unit = { val callback = new ThrottleCallback { @@ -43,7 +44,7 @@ class RequestHandlerHelper( quotaManager.throttle(request.header.clientId(), request.session, callback, throttleTimeMs) } - def handleError(request: RequestChannel.Request, e: Throwable): Unit = { + def handleError(request: Request, e: Throwable): Unit = { val mayThrottle = e.isInstanceOf[ClusterAuthorizationException] || !request.header.apiKey.clusterAction if (mayThrottle) sendErrorResponseMaybeThrottle(request, e) @@ -52,11 +53,11 @@ class RequestHandlerHelper( } private def sendErrorOrCloseConnection( - request: RequestChannel.Request, + request: Request, error: Throwable, throttleMs: Int ): Unit = { - val requestBody = request.body[AbstractRequest] + val requestBody = request.body(classOf[AbstractRequest]) val response = requestBody.getErrorResponse(throttleMs, error) if (response == null) requestChannel.closeConnection(request, requestBody.errorCounts(error)) @@ -64,7 +65,7 @@ class RequestHandlerHelper( requestChannel.sendResponse(request, response, None) } - def sendForwardedResponse(request: RequestChannel.Request, + def sendForwardedResponse(request: Request, response: AbstractResponse): Unit = { // For requests forwarded to the controller, we take the maximum of the local // request throttle and the throttle sent by the controller in the response. @@ -79,7 +80,7 @@ class RequestHandlerHelper( // Throttle the channel if the request quota is enabled but has been violated. Regardless of throttling, send the // response immediately. def sendMaybeThrottle( - request: RequestChannel.Request, + request: Request, response: AbstractResponse ): Unit = { val throttleTimeMs = maybeRecordAndGetThrottleTimeMs(request) @@ -90,7 +91,7 @@ class RequestHandlerHelper( requestChannel.sendResponse(request, response, None) } - def sendResponseMaybeThrottle(request: RequestChannel.Request, + def sendResponseMaybeThrottle(request: Request, createResponse: Int => AbstractResponse): Unit = { val throttleTimeMs = maybeRecordAndGetThrottleTimeMs(request) // Only throttle non-forwarded requests @@ -99,7 +100,7 @@ class RequestHandlerHelper( requestChannel.sendResponse(request, createResponse(throttleTimeMs), None) } - def sendErrorResponseMaybeThrottle(request: RequestChannel.Request, error: Throwable): Unit = { + def sendErrorResponseMaybeThrottle(request: Request, error: Throwable): Unit = { val throttleTimeMs = maybeRecordAndGetThrottleTimeMs(request) // Only throttle non-forwarded requests or cluster authorization failures if (error.isInstanceOf[ClusterAuthorizationException] || !request.isForwarded) @@ -107,9 +108,9 @@ class RequestHandlerHelper( sendErrorOrCloseConnection(request, error, throttleTimeMs) } - def maybeRecordAndGetThrottleTimeMs(request: RequestChannel.Request): Int = { + def maybeRecordAndGetThrottleTimeMs(request: Request): Int = { val throttleTimeMs = quotas.request.maybeRecordAndGetThrottleTimeMs(request, time.milliseconds()) - request.apiThrottleTimeMs = throttleTimeMs + request.apiThrottleTimeMs(throttleTimeMs) throttleTimeMs } @@ -119,7 +120,7 @@ class RequestHandlerHelper( */ def sendResponseMaybeThrottleWithControllerQuota( controllerMutationQuota: ControllerMutationQuota, - request: RequestChannel.Request, + request: Request, response: AbstractResponse ): Unit = { val timeMs = time.milliseconds @@ -128,7 +129,7 @@ class RequestHandlerHelper( val maxThrottleTimeMs = Math.max(controllerThrottleTimeMs, requestThrottleTimeMs) // Only throttle non-forwarded requests if (maxThrottleTimeMs > 0 && !request.isForwarded) { - request.apiThrottleTimeMs = maxThrottleTimeMs + request.apiThrottleTimeMs(maxThrottleTimeMs) if (controllerThrottleTimeMs > requestThrottleTimeMs) { throttle(quotas.controllerMutation, request, controllerThrottleTimeMs) } else { @@ -140,19 +141,19 @@ class RequestHandlerHelper( requestChannel.sendResponse(request, response, None) } - def sendResponseExemptThrottle(request: RequestChannel.Request, + def sendResponseExemptThrottle(request: Request, response: AbstractResponse, onComplete: Option[Send => Unit] = None): Unit = { quotas.request.maybeRecordExempt(request) requestChannel.sendResponse(request, response, onComplete) } - private def sendErrorResponseExemptThrottle(request: RequestChannel.Request, error: Throwable): Unit = { + private def sendErrorResponseExemptThrottle(request: Request, error: Throwable): Unit = { quotas.request.maybeRecordExempt(request) sendErrorOrCloseConnection(request, error, 0) } - def sendNoOpResponseExemptThrottle(request: RequestChannel.Request): Unit = { + def sendNoOpResponseExemptThrottle(request: Request): Unit = { quotas.request.maybeRecordExempt(request) requestChannel.sendNoOpResponse(request) } diff --git a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala index 081fbec3c95d7..2b9ddde709a23 100644 --- a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala +++ b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala @@ -25,6 +25,7 @@ import org.apache.kafka.common.message.{BeginQuorumEpochResponseData, EndQuorumE import org.apache.kafka.common.protocol.{ApiKeys, ApiMessage} import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, BeginQuorumEpochResponse, EndQuorumEpochResponse, FetchResponse, FetchSnapshotResponse, VoteResponse} import org.apache.kafka.common.utils.Time +import org.apache.kafka.network.Request import org.apache.kafka.raft.RaftManager import org.apache.kafka.server.ApiVersionManager import org.apache.kafka.server.common.RequestLocal @@ -39,7 +40,7 @@ class TestRaftRequestHandler( apiVersionManager: ApiVersionManager ) extends ApiRequestHandler with Logging { - override def handle(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { + override def handle(request: Request, requestLocal: RequestLocal): Unit = { try { trace(s"Handling request:${request.requestDesc(true)} with context ${request.context}") request.header.apiKey match { @@ -56,44 +57,44 @@ class TestRaftRequestHandler( case e: Throwable => error(s"Unexpected error handling request ${request.requestDesc(true)} " + s"with context ${request.context}", e) - val errorResponse = request.body[AbstractRequest].getErrorResponse(e) + val errorResponse = request.body(classOf[AbstractRequest]).getErrorResponse(e) requestChannel.sendResponse(request, errorResponse, None) } finally { // The local completion time may be set while processing the request. Only record it if it's unset. if (request.apiLocalCompleteTimeNanos < 0) - request.apiLocalCompleteTimeNanos = time.nanoseconds + request.apiLocalCompleteTimeNanos(time.nanoseconds) } } - private def handleApiVersions(request: RequestChannel.Request): Unit = { + private def handleApiVersions(request: Request): Unit = { requestChannel.sendResponse(request, apiVersionManager.apiVersionResponse(0, request.header.apiVersion() < 4), None) } - private def handleVote(request: RequestChannel.Request): Unit = { + private def handleVote(request: Request): Unit = { handle(request, response => new VoteResponse(response.asInstanceOf[VoteResponseData])) } - private def handleBeginQuorumEpoch(request: RequestChannel.Request): Unit = { + private def handleBeginQuorumEpoch(request: Request): Unit = { handle(request, response => new BeginQuorumEpochResponse(response.asInstanceOf[BeginQuorumEpochResponseData])) } - private def handleEndQuorumEpoch(request: RequestChannel.Request): Unit = { + private def handleEndQuorumEpoch(request: Request): Unit = { handle(request, response => new EndQuorumEpochResponse(response.asInstanceOf[EndQuorumEpochResponseData])) } - private def handleFetch(request: RequestChannel.Request): Unit = { + private def handleFetch(request: Request): Unit = { handle(request, response => FetchResponse.of(response.asInstanceOf[FetchResponseData])) } - private def handleFetchSnapshot(request: RequestChannel.Request): Unit = { + private def handleFetchSnapshot(request: Request): Unit = { handle(request, response => new FetchSnapshotResponse(response.asInstanceOf[FetchSnapshotResponseData])) } private def handle( - request: RequestChannel.Request, + request: Request, buildResponse: ApiMessage => AbstractResponse ): Unit = { - val requestBody = request.body[AbstractRequest] + val requestBody = request.body(classOf[AbstractRequest]) val future = raftManager.handleRequest( request.context, diff --git a/core/src/test/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandlerTest.java b/core/src/test/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandlerTest.java index 90f2c10d00b63..3f20bcae6c33a 100644 --- a/core/src/test/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandlerTest.java +++ b/core/src/test/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandlerTest.java @@ -17,7 +17,6 @@ package kafka.server.handlers; -import kafka.network.RequestChannel; import kafka.server.AuthHelper; import kafka.server.KafkaConfig; import kafka.utils.TestUtils; @@ -58,6 +57,7 @@ import org.apache.kafka.image.MetadataProvenance; import org.apache.kafka.metadata.KRaftMetadataCache; import org.apache.kafka.metadata.LeaderRecoveryState; +import org.apache.kafka.network.Request; import org.apache.kafka.network.SocketServerConfigs; import org.apache.kafka.network.metrics.RequestChannelMetrics; import org.apache.kafka.raft.KRaftConfigs; @@ -202,7 +202,7 @@ void testDescribeTopicPartitionsRequest() { new DescribeTopicPartitionsRequestData.TopicRequest().setName(unauthorizedTopic) )) ); - RequestChannel.Request request; + Request request; try { request = buildRequest(describeTopicPartitionsRequest, plaintextListener); } catch (Exception e) { @@ -401,7 +401,7 @@ void testDescribeTopicPartitionsRequestWithEdgeCases() { .setCursor(new DescribeTopicPartitionsRequestData.Cursor().setTopicName(authorizedTopic).setPartitionIndex(1)) ); - RequestChannel.Request request; + Request request; try { request = buildRequest(describeTopicPartitionsRequest, plaintextListener); } catch (Exception e) { @@ -497,7 +497,7 @@ void updateKraftMetadataCache(KRaftMetadataCache kRaftMetadataCache, List callback(1)) - request.apiLocalCompleteTimeNanos = time.nanoseconds + request.apiLocalCompleteTimeNanos(time.nanoseconds) } handler.run() assertEquals(startTime, request.requestDequeueTimeNanos) assertEquals(startTime + 2000000, request.apiLocalCompleteTimeNanos) - assertEquals(Some(startTime + 2000000), request.callbackRequestDequeueTimeNanos) - assertEquals(Some(startTime + 3000000), request.callbackRequestCompleteTimeNanos) + assertEquals(startTime + 2000000, request.callbackRequestDequeueTimeNanos.getAsLong) + assertEquals(startTime + 3000000, request.callbackRequestCompleteTimeNanos.getAsLong) } finally { metrics.close() } @@ -231,7 +232,7 @@ class KafkaRequestHandlerTest { }) } - def makeRequest(time: Time, metrics: RequestChannelMetrics): RequestChannel.Request = { + def makeRequest(time: Time, metrics: RequestChannelMetrics): Request = { // Make unsupported API versions request to avoid having to parse a real request val requestHeader = mock(classOf[RequestHeader]) when(requestHeader.apiKey()).thenReturn(ApiKeys.API_VERSIONS) @@ -239,8 +240,7 @@ class KafkaRequestHandlerTest { val context = new RequestContext(requestHeader, "0", mock(classOf[InetAddress]), new KafkaPrincipal("", ""), new ListenerName(""), SecurityProtocol.PLAINTEXT, mock(classOf[ClientInformation]), false) - new RequestChannel.Request(0, context, time.nanoseconds(), - mock(classOf[MemoryPool]), ByteBuffer.allocate(0), metrics) + new Request(0, context, time.nanoseconds(), mock(classOf[MemoryPool]), ByteBuffer.allocate(0), metrics) } def setupBrokerTopicMetrics(systemRemoteStorageEnabled: Boolean = true): BrokerTopicMetrics = { diff --git a/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala b/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala index 582215500b7f9..1c0cbe74fdba2 100644 --- a/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala +++ b/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala @@ -18,7 +18,6 @@ package kafka.network import com.fasterxml.jackson.databind.ObjectMapper -import kafka.network import kafka.server.EnvelopeUtils import org.apache.kafka.clients.admin.AlterConfigOp.OpType import org.apache.kafka.common.config.types.Password @@ -34,7 +33,7 @@ import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.{KafkaPrincipal, KafkaPrincipalSerde, SecurityProtocol} import org.apache.kafka.common.utils.Utils import org.apache.kafka.common.utils.internals.SecurityUtils -import org.apache.kafka.network.RequestConvertToJson +import org.apache.kafka.network.{Request, RequestConvertToJson} import org.apache.kafka.network.metrics.RequestChannelMetrics import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api._ @@ -50,7 +49,6 @@ import java.util.Optional import java.util.concurrent.atomic.AtomicReference import scala.collection.Map import scala.jdk.CollectionConverters._ -import scala.jdk.OptionConverters.RichOption class RequestChannelTest { private val requestChannelMetrics: RequestChannelMetrics = mock(classOf[RequestChannelMetrics]) @@ -80,7 +78,7 @@ class RequestChannelTest { val loggableAlterConfigs = alterConfigsReq.loggableRequest.asInstanceOf[AlterConfigsRequest] val loggedConfig = loggableAlterConfigs.configs.get(resource) assertEquals(expectedValues, toMap(loggedConfig)) - val alterConfigsDesc = RequestConvertToJson.requestDesc(alterConfigsReq.header, alterConfigsReq.requestLog.toJava, alterConfigsReq.isForwarded).toString + val alterConfigsDesc = RequestConvertToJson.requestDesc(alterConfigsReq.header, alterConfigsReq.requestLog, alterConfigsReq.isForwarded).toString assertFalse(alterConfigsDesc.contains(sensitiveValue), s"Sensitive config logged $alterConfigsDesc") } @@ -154,7 +152,7 @@ class RequestChannelTest { val loggableAlterConfigs = req.loggableRequest.asInstanceOf[IncrementalAlterConfigsRequest] val loggedConfig = loggableAlterConfigs.data.resources.find(resource.`type`.id, resource.name).configs assertEquals(expectedValues, toMap(loggedConfig)) - val alterConfigsDesc = RequestConvertToJson.requestDesc(req.header, req.requestLog.toJava, req.isForwarded).toString + val alterConfigsDesc = RequestConvertToJson.requestDesc(req.header, req.requestLog, req.isForwarded).toString assertFalse(alterConfigsDesc.contains(sensitiveValue), s"Sensitive config logged $alterConfigsDesc") } @@ -199,7 +197,7 @@ class RequestChannelTest { @Test def testNonAlterRequestsNotTransformed(): Unit = { val metadataRequest = request(new MetadataRequest.Builder(util.List.of("topic"), true).build()) - assertSame(metadataRequest.body[MetadataRequest], metadataRequest.loggableRequest) + assertSame(metadataRequest.body(classOf[MetadataRequest]), metadataRequest.loggableRequest) } @Test @@ -257,8 +255,8 @@ class RequestChannelTest { ) new CreateTopicsResponse(responseData) } - - private def buildUnwrappedEnvelopeRequest(request: AbstractRequest): RequestChannel.Request = { + + private def buildUnwrappedEnvelopeRequest(request: AbstractRequest): Request = { val wrappedRequest = buildEnvelopeRequest( request, principalSerde, @@ -266,7 +264,7 @@ class RequestChannelTest { System.nanoTime() ) - val unwrappedRequest = new AtomicReference[RequestChannel.Request]() + val unwrappedRequest = new AtomicReference[Request]() EnvelopeUtils.handleEnvelopeRequest( wrappedRequest, @@ -284,7 +282,7 @@ class RequestChannelTest { startTimeNanos: Long, dequeueTimeNanos: Long = -1, fromPrivilegedListener: Boolean = true - ): RequestChannel.Request = { + ): Request = { val clientId = "id" val listenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT) @@ -304,16 +302,16 @@ class RequestChannelTest { KafkaPrincipal.ANONYMOUS, listenerName, SecurityProtocol.PLAINTEXT, ClientInformation.EMPTY, fromPrivilegedListener, Optional.of(principalSerde)) - val envelopRequest = new RequestChannel.Request( - processor = 1, - context = envelopeContext, - startTimeNanos = startTimeNanos, - memoryPool = MemoryPool.NONE, - buffer = envelopeBuffer, - metrics = requestChannelMetrics, - envelope = None + val envelopRequest = new Request( + 1, + envelopeContext, + startTimeNanos, + MemoryPool.NONE, + envelopeBuffer, + requestChannelMetrics, + Optional.empty ) - envelopRequest.requestDequeueTimeNanos = dequeueTimeNanos + envelopRequest.requestDequeueTimeNanos(dequeueTimeNanos) envelopRequest } @@ -327,16 +325,10 @@ class RequestChannelTest { } } - def request(req: AbstractRequest): RequestChannel.Request = { + def request(req: AbstractRequest): Request = { val buffer = req.serializeWithHeader(new RequestHeader(req.apiKey, req.version, "client-id", 1)) val requestContext = newRequestContext(buffer) - new network.RequestChannel.Request(processor = 1, - requestContext, - startTimeNanos = 0, - mock(classOf[MemoryPool]), - buffer, - mock(classOf[RequestChannelMetrics]) - ) + new Request(1, requestContext, 0, mock(classOf[MemoryPool]), buffer, mock(classOf[RequestChannelMetrics])) } private def newRequestContext(buffer: ByteBuffer): RequestContext = { @@ -360,10 +352,10 @@ class RequestChannelTest { } private def buildEnvelopeResponse( - unwrapped: RequestChannel.Request, + unwrapped: Request, response: AbstractResponse ): EnvelopeResponse = { - assertTrue(unwrapped.envelope.isDefined) + assertTrue(unwrapped.envelope.isPresent) val envelope = unwrapped.envelope.get val send = unwrapped.buildResponseSend(response) diff --git a/core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala b/core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala index 400e13dc6bef7..a5566c869b355 100644 --- a/core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala +++ b/core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala @@ -20,14 +20,13 @@ package kafka.network import java.net.InetAddress import java.nio.ByteBuffer import com.fasterxml.jackson.databind.node.{BooleanNode, DoubleNode, JsonNodeFactory, LongNode, ObjectNode, TextNode} -import kafka.network import org.apache.kafka.common.memory.MemoryPool import org.apache.kafka.common.message._ import org.apache.kafka.common.network.{ClientInformation, ListenerName, NetworkSend} import org.apache.kafka.common.protocol.ApiKeys import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} -import org.apache.kafka.network.RequestConvertToJson +import org.apache.kafka.network.{Request, RequestConvertToJson} import org.apache.kafka.network.metrics.RequestChannelMetrics import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -59,9 +58,9 @@ class RequestConvertToJsonTest { val expectedNode = new ObjectNode(JsonNodeFactory.instance) expectedNode.set("isForwarded", if (req.isForwarded) BooleanNode.TRUE else BooleanNode.FALSE) expectedNode.set("requestHeader", RequestConvertToJson.requestHeaderNode(req.header)) - expectedNode.set("request", req.requestLog.getOrElse(new TextNode(""))) + expectedNode.set("request", req.requestLog.orElse(new TextNode(""))) - val actualNode = RequestConvertToJson.requestDesc(req.header, req.requestLog.toJava, req.isForwarded) + val actualNode = RequestConvertToJson.requestDesc(req.header, req.requestLog, req.isForwarded) assertEquals(expectedNode, actualNode) } @@ -84,7 +83,7 @@ class RequestConvertToJsonTest { val temporaryMemoryBytes = 8 val messageConversionsTimeMs = 9 - val expectedNode = RequestConvertToJson.requestDesc(req.header, req.requestLog.toJava, req.isForwarded).asInstanceOf[ObjectNode] + val expectedNode = RequestConvertToJson.requestDesc(req.header, req.requestLog, req.isForwarded).asInstanceOf[ObjectNode] expectedNode.set("response", res.responseLog.getOrElse(new TextNode(""))) expectedNode.set("connection", new TextNode(req.context.connectionId)) expectedNode.set("totalTimeMs", new DoubleNode(totalTimeMs)) @@ -101,23 +100,17 @@ class RequestConvertToJsonTest { expectedNode.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes)) expectedNode.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs)) - val actualNode = RequestConvertToJson.requestDescMetrics(req.header, req.requestLog.toJava, res.responseLog.toJava, req.context, req.session, req.isForwarded, + val actualNode = RequestConvertToJson.requestDescMetrics(req.header, req.requestLog, res.responseLog.toJava, req.context, req.session, req.isForwarded, totalTimeMs, requestQueueTimeMs, apiLocalTimeMs, apiRemoteTimeMs, apiThrottleTimeMs, responseQueueTimeMs, responseSendTimeMs, temporaryMemoryBytes, messageConversionsTimeMs).asInstanceOf[ObjectNode] assertEquals(expectedNode, actualNode) } - def request(req: AbstractRequest): RequestChannel.Request = { + def request(req: AbstractRequest): Request = { val buffer = req.serializeWithHeader(new RequestHeader(req.apiKey, req.version, "client-id", 1)) val requestContext = newRequestContext(buffer) - new network.RequestChannel.Request(processor = 1, - requestContext, - startTimeNanos = 0, - mock(classOf[MemoryPool]), - buffer, - mock(classOf[RequestChannelMetrics]) - ) + new Request(1, requestContext, 0, mock(classOf[MemoryPool]), buffer, mock(classOf[RequestChannelMetrics])) } private def newRequestContext(buffer: ByteBuffer): RequestContext = { diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index 2e0929da8ae20..978fc39715fc1 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -36,8 +36,7 @@ import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} import org.apache.kafka.common.security.scram.internals.ScramMechanism import org.apache.kafka.common.utils._ import org.apache.kafka.common.utils.internals.{AppInfoParser, LogContext} -import org.apache.kafka.network.RequestConvertToJson -import org.apache.kafka.network.SocketServerConfigs +import org.apache.kafka.network.{CallbackRequest, Request, RequestConvertToJson, ShutdownRequest, SocketServerConfigs, WakeupRequest} import org.apache.kafka.security.CredentialProvider import org.apache.kafka.server.{ApiVersionManager, SimpleApiVersionManager} import org.apache.kafka.server.common.{FinalizedFeatures, MetadataVersion} @@ -146,12 +145,12 @@ class SocketServerTest { response } - private def receiveRequest(channel: RequestChannel, timeout: Long = 2000L): RequestChannel.Request = { + private def receiveRequest(channel: RequestChannel, timeout: Long = 2000L): Request = { channel.receiveRequest(timeout) match { - case request: RequestChannel.Request => request - case RequestChannel.WakeupRequest => throw new AssertionError("Unexpected wakeup received") - case request: RequestChannel.CallbackRequest => throw new AssertionError("Unexpected callback received") - case RequestChannel.ShutdownRequest => throw new AssertionError("Unexpected shutdown received") + case request: Request => request + case _: WakeupRequest => throw new AssertionError("Unexpected wakeup received") + case _: CallbackRequest => throw new AssertionError("Unexpected callback received") + case _: ShutdownRequest => throw new AssertionError("Unexpected shutdown received") case null => throw new AssertionError("receiveRequest timed out") } } @@ -161,14 +160,14 @@ class SocketServerTest { processRequest(channel, receiveRequest(channel)) } - def processRequest(channel: RequestChannel, request: RequestChannel.Request): Unit = { - val byteBuffer = request.body[AbstractRequest].serializeWithHeader(request.header) + def processRequest(channel: RequestChannel, request: Request): Unit = { + val byteBuffer = request.body(classOf[AbstractRequest]).serializeWithHeader(request.header) val send = new NetworkSend(request.context.connectionId, ByteBufferSend.sizePrefixed(byteBuffer)) val headerLog = RequestConvertToJson.requestHeaderNode(request.header) channel.sendResponse(new RequestChannel.SendResponse(request, send, Some(headerLog), None)) } - def processRequestNoOpResponse(channel: RequestChannel, request: RequestChannel.Request): Unit = { + def processRequestNoOpResponse(channel: RequestChannel, request: Request): Unit = { channel.sendNoOpResponse(request) } @@ -223,7 +222,7 @@ class SocketServerTest { (socket, request.context.connectionId) } - def sendAndReceiveRequest(socket: Socket, server: SocketServer): RequestChannel.Request = { + def sendAndReceiveRequest(socket: Socket, server: SocketServer): Request = { sendRequest(socket, producerRequestBytes()) receiveRequest(server.dataPlaneRequestChannel) } @@ -549,7 +548,7 @@ class SocketServerTest { private def makeSocketWithBufferedRequests(server: SocketServer, serverSelector: Selector, proxyServer: ProxyServer, - numBufferedRequests: Int = 2): (Socket, RequestChannel.Request) = { + numBufferedRequests: Int = 2): (Socket, Request) = { val requestBytes = producerRequestBytes() val socket = sslClientSocket(proxyServer.localPort) @@ -578,7 +577,7 @@ class SocketServerTest { */ private def makeChannelWithBufferedRequestsAndCloseRemote(server: TestableSocketServer, serverSelector: TestableSelector, - makeClosing: Boolean = false): RequestChannel.Request = { + makeClosing: Boolean = false): Request = { val proxyServer = new ProxyServer(server) try { @@ -599,15 +598,15 @@ class SocketServerTest { } } - def sendRequestsReceiveOne(server: SocketServer, socket: Socket, requestBytes: Array[Byte], numRequests: Int): RequestChannel.Request = { + def sendRequestsReceiveOne(server: SocketServer, socket: Socket, requestBytes: Array[Byte], numRequests: Int): Request = { (1 to numRequests).foreach(i => sendRequest(socket, requestBytes, flush = i == numRequests)) receiveRequest(server.dataPlaneRequestChannel) } private def closeSocketWithPendingRequest(server: SocketServer, - createSocket: () => Socket): RequestChannel.Request = { + createSocket: () => Socket): Request = { - def maybeReceiveRequest(): Option[RequestChannel.Request] = { + def maybeReceiveRequest(): Option[Request] = { try { Some(receiveRequest(server.dataPlaneRequestChannel, timeout = 1000)) } catch { @@ -615,7 +614,7 @@ class SocketServerTest { } } - def closedChannelWithPendingRequest(): Option[RequestChannel.Request] = { + def closedChannelWithPendingRequest(): Option[Request] = { val socket = createSocket.apply() val req1 = sendRequestsReceiveOne(server, socket, producerRequestBytes(), numRequests = 100) processRequestNoOpResponse(server.dataPlaneRequestChannel, req1) @@ -636,13 +635,13 @@ class SocketServerTest { // Prepares test setup for throttled channel tests. throttlingDone controls whether or not throttling has completed // in quota manager. def throttledChannelTestSetUp(socket: Socket, serializedBytes: Array[Byte], noOpResponse: Boolean, - throttlingInProgress: Boolean): RequestChannel.Request = { + throttlingInProgress: Boolean): Request = { sendRequest(socket, serializedBytes) // Mimic a primitive request handler that fetches the request from RequestChannel and place a response with a // throttled channel. val request = receiveRequest(server.dataPlaneRequestChannel) - val byteBuffer = request.body[AbstractRequest].serializeWithHeader(request.header) + val byteBuffer = request.body(classOf[AbstractRequest]).serializeWithHeader(request.header) val send = new NetworkSend(request.context.connectionId, ByteBufferSend.sizePrefixed(byteBuffer)) val channelThrottlingCallback = new ThrottleCallback { @@ -666,10 +665,10 @@ class SocketServerTest { request } - def openChannel(request: RequestChannel.Request, server: SocketServer = this.server): Option[KafkaChannel] = + def openChannel(request: Request, server: SocketServer = this.server): Option[KafkaChannel] = server.dataPlaneAcceptor(listener).get.processors(0).channel(request.context.connectionId) - def openOrClosingChannel(request: RequestChannel.Request, server: SocketServer = this.server): Option[KafkaChannel] = + def openOrClosingChannel(request: Request, server: SocketServer = this.server): Option[KafkaChannel] = server.dataPlaneAcceptor(listener).get.processors(0).openOrClosingChannel(request.context.connectionId) @Test diff --git a/core/src/test/scala/unit/kafka/server/AuthHelperTest.scala b/core/src/test/scala/unit/kafka/server/AuthHelperTest.scala index eb1eee15cc5c7..1f08046a1134b 100644 --- a/core/src/test/scala/unit/kafka/server/AuthHelperTest.scala +++ b/core/src/test/scala/unit/kafka/server/AuthHelperTest.scala @@ -17,11 +17,9 @@ package kafka.server -import kafka.network.RequestChannel.Request -import org.apache.kafka.clients.admin.EndpointType - import java.net.InetAddress import java.util +import org.apache.kafka.clients.admin.EndpointType import org.apache.kafka.common.acl.AclOperation import org.apache.kafka.common.internals.Plugin import org.apache.kafka.common.message.{DescribeClusterRequestData, DescribeClusterResponseData} @@ -31,6 +29,7 @@ import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests.{DescribeClusterRequest, RequestContext, RequestHeader} import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourceType} import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} +import org.apache.kafka.network.Request import org.apache.kafka.server.authorizer.{Action, AuthorizationResult, Authorizer} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test @@ -56,7 +55,7 @@ object AuthHelperTest { ClientInformation.EMPTY, false) val request: Request = mock(classOf[Request]) - when(request.body[DescribeClusterRequest]).thenReturn( + when(request.body(classOf[DescribeClusterRequest])).thenReturn( new DescribeClusterRequest(data, requestVersion.toShort)) when(request.context).thenReturn(requestContext) when(request.header).thenReturn(requestContext.header) diff --git a/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala index e25733a6f0dc4..294d5f279887b 100644 --- a/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala @@ -19,7 +19,6 @@ package kafka.server import java.net.InetAddress import java.util import java.util.Collections -import kafka.network.RequestChannel import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.memory.MemoryPool import org.apache.kafka.common.metrics.{MetricConfig, Metrics} @@ -29,7 +28,7 @@ import org.apache.kafka.common.requests.FetchRequest.PartitionData import org.apache.kafka.common.requests.{AbstractRequest, FetchRequest, RequestContext, RequestHeader} import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} import org.apache.kafka.common.utils.MockTime -import org.apache.kafka.network.Session +import org.apache.kafka.network.{Request, Session} import org.apache.kafka.network.metrics.RequestChannelMetrics import org.apache.kafka.server.quota.{ClientQuotaManager, ThrottleCallback} import org.junit.jupiter.api.AfterEach @@ -54,7 +53,7 @@ class BaseClientQuotaManagerTest { } protected def buildRequest[T <: AbstractRequest](builder: AbstractRequest.Builder[T], - listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, RequestChannel.Request) = { + listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, Request) = { val request = builder.build() val buffer = request.serializeWithHeader(new RequestHeader(builder.apiKey, request.version, "", 0)) @@ -64,8 +63,7 @@ class BaseClientQuotaManagerTest { val header = RequestHeader.parse(buffer) val context = new RequestContext(header, "1", InetAddress.getLocalHost, KafkaPrincipal.ANONYMOUS, listenerName, SecurityProtocol.PLAINTEXT, ClientInformation.EMPTY, false) - (request, new RequestChannel.Request(processor = 1, context = context, startTimeNanos = 0, MemoryPool.NONE, buffer, - requestChannelMetrics)) + (request, new Request(1, context, 0, MemoryPool.NONE, buffer, requestChannelMetrics)) } protected def buildSession(user: String): Session = { diff --git a/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala b/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala index ed1465faa6ff9..7f30d8f297169 100644 --- a/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala @@ -52,7 +52,7 @@ import org.apache.kafka.controller.ControllerRequestContextUtil.ANONYMOUS_CONTEX import org.apache.kafka.controller.{Controller, ControllerRequestContext, ResultOrError} import org.apache.kafka.image.publisher.ControllerRegistrationsPublisher import org.apache.kafka.metadata.KRaftMetadataCache -import org.apache.kafka.network.{Session, SocketServerConfigs} +import org.apache.kafka.network.{Request, Session, SocketServerConfigs} import org.apache.kafka.network.metrics.RequestChannelMetrics import org.apache.kafka.raft.{KRaftConfigs, QuorumConfig, RaftManager} import org.apache.kafka.server.SimpleApiVersionManager @@ -182,7 +182,7 @@ class ControllerApisTest { } /** - * Build a RequestChannel.Request from the AbstractRequest + * Build a Request from the AbstractRequest * * @param request - AbstractRequest * @param listenerName - Default listener for the RequestChannel @@ -192,15 +192,14 @@ class ControllerApisTest { private def buildRequest[T <: AbstractRequest]( request: AbstractRequest, listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT) - ): RequestChannel.Request = { + ): Request = { val buffer = request.serializeWithHeader(new RequestHeader(request.apiKey, request.version, clientID, 0)) // read the header from the buffer first so that the body can be read next from the Request constructor val header = RequestHeader.parse(buffer) val context = new RequestContext(header, "1", InetAddress.getLocalHost, KafkaPrincipal.ANONYMOUS, listenerName, SecurityProtocol.PLAINTEXT, ClientInformation.EMPTY, false) - new RequestChannel.Request(processor = 1, context = context, startTimeNanos = 0, MemoryPool.NONE, buffer, - requestChannelMetrics) + new Request(1, context, 0, MemoryPool.NONE, buffer, requestChannelMetrics) } def createDenyAllAuthorizer(): Plugin[Authorizer] = { diff --git a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala index 46815eaaa4b02..a8936ada08231 100644 --- a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala @@ -20,8 +20,6 @@ import java.net.InetAddress import java.nio.ByteBuffer import java.util.Optional import java.util.concurrent.atomic.AtomicReference -import kafka.network -import kafka.network.RequestChannel import org.apache.kafka.clients.{MockClient, NodeApiVersions} import org.apache.kafka.clients.MockClient.RequestMatcher import org.apache.kafka.common.Node @@ -35,6 +33,7 @@ import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, AlterConfigsRequest, AlterConfigsResponse, EnvelopeRequest, EnvelopeResponse, RequestContext, RequestHeader, RequestTestUtils} import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} import org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder +import org.apache.kafka.network.Request import org.apache.kafka.network.metrics.RequestChannelMetrics import org.apache.kafka.server.util.MockTime import org.apache.kafka.server.ControllerInformation @@ -269,7 +268,7 @@ class ForwardingManagerTest { ) val buffer = body.serializeWithHeader(header) - // Fast-forward buffer to start of the request as `RequestChannel.Request` expects + // Fast-forward buffer to start of the request as `Request` expects RequestHeader.parse(buffer) (header, buffer) @@ -279,7 +278,7 @@ class ForwardingManagerTest { requestHeader: RequestHeader, requestBuffer: ByteBuffer, principal: KafkaPrincipal - ): RequestChannel.Request = { + ): Request = { val requestContext = new RequestContext( requestHeader, "1", @@ -293,14 +292,8 @@ class ForwardingManagerTest { Optional.of(principalBuilder) ) - new network.RequestChannel.Request( - processor = 1, - context = requestContext, - startTimeNanos = time.nanoseconds(), - memoryPool = MemoryPool.NONE, - buffer = requestBuffer, - metrics = new RequestChannelMetrics(ListenerType.CONTROLLER), - envelope = None + new Request(1, requestContext, time.nanoseconds(), + MemoryPool.NONE, requestBuffer, new RequestChannelMetrics(ListenerType.CONTROLLER) ) } diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 9926a1bafe5a3..b5d84c0527aa2 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -87,7 +87,7 @@ import org.apache.kafka.coordinator.share.{ShareCoordinator, ShareCoordinatorTes import org.apache.kafka.coordinator.transaction.TransactionLogConfig import org.apache.kafka.image.{MetadataDelta, MetadataImage, MetadataProvenance} import org.apache.kafka.metadata.{ConfigRepository, KRaftMetadataCache, MetadataCache, MetadataCacheFixtures, MockConfigRepository} -import org.apache.kafka.network.Session +import org.apache.kafka.network.{Request, Session} import org.apache.kafka.network.metrics.{RequestChannelMetrics, RequestMetrics} import org.apache.kafka.raft.{KRaftConfigs, QuorumConfig} import org.apache.kafka.security.authorizer.AclEntry @@ -126,6 +126,7 @@ import java.util.function.Consumer import java.util.{Comparator, Optional, OptionalInt, OptionalLong, Properties} import scala.collection.{Map, Seq, mutable} import scala.jdk.CollectionConverters._ +import scala.jdk.OptionConverters._ class KafkaApisTest extends Logging { private val requestChannel: RequestChannel = mock(classOf[RequestChannel]) @@ -394,7 +395,7 @@ class KafkaApisTest extends Logging { .build(requestHeader.apiVersion) val request = buildRequest(describeConfigsRequest, requestHeader = Option(requestHeader)) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) createKafkaApis(authorizer = Some(authorizer), configRepository = configRepository) @@ -555,7 +556,7 @@ class KafkaApisTest extends Logging { val apiRequest = requestBuilder.build(topicHeader.apiVersion) val request = buildRequest(apiRequest) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) val forwardCallback: ArgumentCaptor[Option[AbstractResponse] => Unit] = ArgumentCaptor.forClass(classOf[Option[AbstractResponse] => Unit]) @@ -789,7 +790,7 @@ class KafkaApisTest extends Logging { .setKey(key)) } val request = buildRequest(findCoordinatorRequestBuilder.build(requestHeader.apiVersion)) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) val capturedRequest = verifyTopicCreation(topicName, enableAutoTopicCreation = true, isInternal = true, request) @@ -949,7 +950,7 @@ class KafkaApisTest extends Logging { ).build(requestHeader.apiVersion) val request = buildRequest(metadataRequest) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) val capturedRequest = verifyTopicCreation(topicName, enableAutoTopicCreation, isInternal, request) @@ -975,7 +976,7 @@ class KafkaApisTest extends Logging { private def verifyTopicCreation(topicName: String, enableAutoTopicCreation: Boolean, isInternal: Boolean, - request: RequestChannel.Request): ArgumentCaptor[Option[RequestContext]] = { + request: Request): ArgumentCaptor[Option[RequestContext]] = { val capturedRequest: ArgumentCaptor[Option[RequestContext]] = ArgumentCaptor.forClass(classOf[Option[RequestContext]]) if (enableAutoTopicCreation) { @@ -1347,7 +1348,7 @@ class KafkaApisTest extends Logging { .setTopics(TxnOffsetCommitRequest.getTopics(util.Map.of(invalidTopicPartition, partitionOffsetCommitData))) val offsetCommitRequest = TxnOffsetCommitRequest.Builder.forTopicNames(data, true).build() val request = buildRequest(offsetCommitRequest) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) val kafkaApis = createKafkaApis() try { @@ -2441,7 +2442,7 @@ class KafkaApisTest extends Logging { any() )).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.INVALID_PRODUCER_EPOCH)))) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) when(clientQuotaManager.maybeRecordAndGetThrottleTimeMs( any[Session](), anyString, anyDouble, anyLong)).thenReturn(0) @@ -2501,7 +2502,7 @@ class KafkaApisTest extends Logging { // Create argument captors to verify session information passed to quota managers val sessionCaptorFetch = ArgumentCaptor.forClass(classOf[Session]) val clientIdCaptor = ArgumentCaptor.forClass(classOf[String]) - val requestCaptor = ArgumentCaptor.forClass(classOf[RequestChannel.Request]) + val requestCaptor = ArgumentCaptor.forClass(classOf[Request]) // Mock quota manager responses and capture arguments when(quotas.fetch.maybeRecordAndGetThrottleTimeMs( @@ -2548,7 +2549,7 @@ class KafkaApisTest extends Logging { verify(quotas.fetch, times(1)).maybeRecordAndGetThrottleTimeMs( any[Session](), anyString, anyDouble, anyLong) verify(quotas.request, times(1)).maybeRecordAndGetThrottleTimeMs( - any[RequestChannel.Request](), anyLong) + any[Request](), anyLong) // Verify the Session data passed to fetch quota manager is exactly what was defined in the test val capturedSession = sessionCaptorFetch.getValue @@ -2596,7 +2597,7 @@ class KafkaApisTest extends Logging { .setErrorCode(Errors.NONE.code)))) // Create argument captors to verify session information passed to quota managers - val requestCaptor = ArgumentCaptor.forClass(classOf[RequestChannel.Request]) + val requestCaptor = ArgumentCaptor.forClass(classOf[Request]) // Mock quota manager responses and capture arguments // For ShareAcknowledge, we only verify Request quota (not fetch quota) @@ -2647,7 +2648,7 @@ class KafkaApisTest extends Logging { // Verify that request quota method was called verify(quotas.request, times(1)).maybeRecordAndGetThrottleTimeMs( - any[RequestChannel.Request](), anyLong) + any[Request](), anyLong) // Verify that fetch quota method was NOT called (ShareAcknowledge only uses request quota) verify(quotas.fetch, times(0)).maybeRecordAndGetThrottleTimeMs( @@ -2708,7 +2709,7 @@ class KafkaApisTest extends Logging { // Create argument captors to verify session information passed to quota managers val sessionCaptorFetch = ArgumentCaptor.forClass(classOf[Session]) val clientIdCaptor = ArgumentCaptor.forClass(classOf[String]) - val requestCaptor = ArgumentCaptor.forClass(classOf[RequestChannel.Request]) + val requestCaptor = ArgumentCaptor.forClass(classOf[Request]) // Mock quota manager responses and capture arguments when(quotas.fetch.maybeRecordAndGetThrottleTimeMs( @@ -2764,7 +2765,7 @@ class KafkaApisTest extends Logging { verify(quotas.fetch, times(1)).maybeRecordAndGetThrottleTimeMs( any[Session](), anyString, anyDouble, anyLong) verify(quotas.request, times(1)).maybeRecordAndGetThrottleTimeMs( - any[RequestChannel.Request](), anyLong) + any[Request](), anyLong) // Verify the Session data passed to fetch quota manager is exactly what was defined in the test val capturedSession = sessionCaptorFetch.getValue @@ -2839,7 +2840,7 @@ class KafkaApisTest extends Logging { when(partition.leaderReplicaIdOpt).thenAnswer(_ => Some(newLeaderId)) when(partition.getLeaderEpoch).thenAnswer(_ => newLeaderEpoch) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) when(clientQuotaManager.maybeRecordAndGetThrottleTimeMs( any[Session](), anyString, anyDouble, anyLong)).thenReturn(0) @@ -2908,7 +2909,7 @@ class KafkaApisTest extends Logging { when(replicaManager.getPartitionOrError(tp.topicPartition())).thenAnswer(_ => Left(Errors.UNKNOWN_TOPIC_OR_PARTITION)) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) when(clientQuotaManager.maybeRecordAndGetThrottleTimeMs( any[Session](), anyString, anyDouble, anyLong)).thenReturn(0) @@ -2980,7 +2981,7 @@ class KafkaApisTest extends Logging { when(replicaManager.getPartitionOrError(tp.topicPartition)).thenAnswer(_ => Left(Errors.UNKNOWN_TOPIC_OR_PARTITION)) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) when(clientQuotaManager.maybeRecordAndGetThrottleTimeMs( any[Session](), anyString, anyDouble, anyLong)).thenReturn(0) @@ -3075,7 +3076,7 @@ class KafkaApisTest extends Logging { ).build() val request = buildRequest(addPartitionsToTxnRequest) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) val kafkaApis = createKafkaApis() try { @@ -4230,7 +4231,7 @@ class KafkaApisTest extends Logging { val listOffsetRequest = ListOffsetsRequest.Builder.forConsumer(true, isolationLevel) .setTargetTimes(targetTimes).build() val request = buildRequest(listOffsetRequest) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) kafkaApis = createKafkaApis() kafkaApis.handleListOffsetRequest(request) @@ -4397,7 +4398,7 @@ class KafkaApisTest extends Logging { // 4. Send TopicMetadataReq using topicId val metadataReqByTopicId = MetadataRequest.Builder.forTopicIds(util.Set.of(authorizedTopicId, unauthorizedTopicId)).build() val repByTopicId = buildRequest(metadataReqByTopicId, plaintextListener) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) kafkaApis = createKafkaApis(authorizer = Some(authorizer)) kafkaApis.handleTopicMetadataRequest(repByTopicId) @@ -8882,7 +8883,7 @@ class KafkaApisTest extends Logging { @ParameterizedTest @ApiKeyVersionsSource(apiKey = ApiKeys.LEAVE_GROUP) def testHandleLeaveGroupWithMultipleMembers(version: Short): Unit = { - def makeRequest(version: Short): RequestChannel.Request = { + def makeRequest(version: Short): Request = { buildRequest(new LeaveGroupRequest.Builder( "group", util.List.of( @@ -9066,7 +9067,7 @@ class KafkaApisTest extends Logging { val fooId = Uuid.randomUuid() addTopicToMetadataCache(foo, topicId = fooId, numPartitions = 2) - def makeRequest(version: Short): RequestChannel.Request = { + def makeRequest(version: Short): Request = { buildRequest( OffsetFetchRequest.Builder.forTopicIdsOrNames( new OffsetFetchRequestData() @@ -9229,7 +9230,7 @@ class KafkaApisTest extends Logging { val barId = Uuid.randomUuid() addTopicToMetadataCache(foo, topicId = fooId, numPartitions = 2) - def makeRequest(version: Short): RequestChannel.Request = { + def makeRequest(version: Short): Request = { buildRequest( OffsetFetchRequest.Builder.forTopicIdsOrNames( new OffsetFetchRequestData() @@ -9366,7 +9367,7 @@ class KafkaApisTest extends Logging { // The single group builder does not support topic ids. @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_FETCH, toVersion = 9) def testHandleOffsetFetchWithSingleGroup(version: Short): Unit = { - def makeRequest(version: Short): RequestChannel.Request = { + def makeRequest(version: Short): Request = { buildRequest(OffsetFetchRequest.Builder.forTopicNames( new OffsetFetchRequestData() .setRequireStable(false) @@ -9450,7 +9451,7 @@ class KafkaApisTest extends Logging { val fooId = Uuid.randomUuid() addTopicToMetadataCache(foo, topicId = fooId, numPartitions = 2) - def makeRequest(version: Short): RequestChannel.Request = { + def makeRequest(version: Short): Request = { buildRequest(OffsetFetchRequest.Builder.forTopicIdsOrNames( new OffsetFetchRequestData() .setRequireStable(false) @@ -9549,7 +9550,7 @@ class KafkaApisTest extends Logging { addTopicToMetadataCache(foo, topicId = fooId, numPartitions = 2) addTopicToMetadataCache(bar, topicId = barId, numPartitions = 2) - def makeRequest(version: Short): RequestChannel.Request = { + def makeRequest(version: Short): Request = { buildRequest( OffsetFetchRequest.Builder.forTopicIdsOrNames( new OffsetFetchRequestData() @@ -9743,7 +9744,7 @@ class KafkaApisTest extends Logging { addTopicToMetadataCache(foo, topicId = fooId, numPartitions = 2) addTopicToMetadataCache(bar, topicId = barId, numPartitions = 2) - def makeRequest(version: Short): RequestChannel.Request = { + def makeRequest(version: Short): Request = { buildRequest( OffsetFetchRequest.Builder.forTopicIdsOrNames( new OffsetFetchRequestData() @@ -10285,7 +10286,7 @@ class KafkaApisTest extends Logging { listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT), fromPrivilegedListener: Boolean = false, requestHeader: Option[RequestHeader] = None, - requestMetrics: RequestChannelMetrics = requestChannelMetrics): RequestChannel.Request = { + requestMetrics: RequestChannelMetrics = requestChannelMetrics): Request = { buildRequest(request, new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "Alice"), InetAddress.getLocalHost, listenerName, fromPrivilegedListener, requestHeader, requestMetrics) } @@ -10296,7 +10297,7 @@ class KafkaApisTest extends Logging { listenerName: ListenerName, fromPrivilegedListener: Boolean, requestHeader: Option[RequestHeader], - requestMetrics: RequestChannelMetrics): RequestChannel.Request = { + requestMetrics: RequestChannelMetrics): Request = { val buffer = request.serializeWithHeader( requestHeader.getOrElse(new RequestHeader(request.apiKey, request.version, clientId, 0))) @@ -10309,12 +10310,11 @@ class KafkaApisTest extends Logging { val context = new RequestContext(header, "1", clientAddress, Optional.empty(), principal, listenerName, SecurityProtocol.SSL, ClientInformation.EMPTY, fromPrivilegedListener, Optional.of(kafkaPrincipalSerde)) - new RequestChannel.Request(processor = 1, context = context, startTimeNanos = 0, MemoryPool.NONE, buffer, - requestMetrics, envelope = None) + new Request(1, context, 0, MemoryPool.NONE, buffer, requestMetrics) } private def verifyNoThrottling[T <: AbstractResponse]( - request: RequestChannel.Request + request: Request ): T = { val capturedResponse: ArgumentCaptor[AbstractResponse] = ArgumentCaptor.forClass(classOf[AbstractResponse]) verify(requestChannel).sendResponse( @@ -10335,7 +10335,7 @@ class KafkaApisTest extends Logging { } private def verifyNoThrottlingAndUpdateMetrics[T <: AbstractResponse]( - request: RequestChannel.Request + request: Request ): T = { val capturedResponse: ArgumentCaptor[AbstractResponse] = ArgumentCaptor.forClass(classOf[AbstractResponse]) verify(requestChannel).sendResponse( @@ -10353,10 +10353,10 @@ class KafkaApisTest extends Logging { val sendResponse = new RequestChannel.SendResponse( request, request.buildResponseSend(response), - request.responseNode(response), + request.responseNode(response).toScala, None ) - request.updateRequestMetrics(time.milliseconds(), sendResponse) + request.updateRequestMetrics(time.milliseconds(), sendResponse.responseLog.toJava) AbstractResponse.parseResponse( request.context.header.apiKey, @@ -10439,7 +10439,7 @@ class KafkaApisTest extends Logging { reset(replicaManager, clientRequestQuotaManager, requestChannel) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) val t0p0 = new TopicPartition("t0", 0) val t0p1 = new TopicPartition("t0", 1) @@ -10562,7 +10562,7 @@ class KafkaApisTest extends Logging { val describeProducersRequest = new DescribeProducersRequest.Builder(data).build() val request = buildRequest(describeProducersRequest) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) kafkaApis = createKafkaApis(authorizer = Some(authorizer)) kafkaApis.handleDescribeProducersRequest(request) @@ -10603,7 +10603,7 @@ class KafkaApisTest extends Logging { .setTransactionalIds(util.List.of("foo", "bar")) val describeTransactionsRequest = new DescribeTransactionsRequest.Builder(data).build() val request = buildRequest(describeTransactionsRequest) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) def buildExpectedActions(transactionalId: String): util.List[Action] = { @@ -10654,7 +10654,7 @@ class KafkaApisTest extends Logging { .setTransactionalIds(util.List.of(transactionalId)) val describeTransactionsRequest = new DescribeTransactionsRequest.Builder(data).build() val request = buildRequest(describeTransactionsRequest) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) def expectDescribe( @@ -10720,7 +10720,7 @@ class KafkaApisTest extends Logging { val data = new ListTransactionsRequestData() val listTransactionsRequest = new ListTransactionsRequest.Builder(data).build() val request = buildRequest(listTransactionsRequest) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) when(txnCoordinator.handleListTransactions(Set.empty[Long], Set.empty[String], -1L, null)) @@ -10740,7 +10740,7 @@ class KafkaApisTest extends Logging { val data = new ListTransactionsRequestData().setTransactionalIdPattern("my.*") val listTransactionsRequest = new ListTransactionsRequest.Builder(data).build() val request = buildRequest(listTransactionsRequest) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) val transactionStates = new util.ArrayList[ListTransactionsResponseData.TransactionState]() @@ -10784,7 +10784,7 @@ class KafkaApisTest extends Logging { def testEmptyLegacyAlterConfigsRequestWithKRaft(): Unit = { val request = buildRequest(new AlterConfigsRequest(new AlterConfigsRequestData(), 1.toShort)) metadataCache = new KRaftMetadataCache(brokerId, () => KRaftVersion.KRAFT_VERSION_0) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) kafkaApis = createKafkaApis() kafkaApis.handleAlterConfigsRequest(request) @@ -10804,7 +10804,7 @@ class KafkaApisTest extends Logging { setName("foo"). setValue(null))))))), 1.toShort)) metadataCache = new KRaftMetadataCache(brokerId, () => KRaftVersion.KRAFT_VERSION_0) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) kafkaApis = createKafkaApis() kafkaApis.handleAlterConfigsRequest(request) @@ -10827,7 +10827,7 @@ class KafkaApisTest extends Logging { ) val request = buildRequest(alterConfigsRequest) metadataCache = new KRaftMetadataCache(brokerId, () => KRaftVersion.KRAFT_VERSION_0) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) kafkaApis = createKafkaApis() kafkaApis.handleIncrementalAlterConfigsRequest(request) @@ -10855,7 +10855,7 @@ class KafkaApisTest extends Logging { ) val request = buildRequest(alterConfigsRequest) metadataCache = new KRaftMetadataCache(brokerId, () => KRaftVersion.KRAFT_VERSION_0) - when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[Request](), any[Long])).thenReturn(0) kafkaApis = createKafkaApis() kafkaApis.handleIncrementalAlterConfigsRequest(request) diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/KRaftMetadataRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/KRaftMetadataRequestBenchmark.java index c062089aa7e70..434acfc859e84 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/KRaftMetadataRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/KRaftMetadataRequestBenchmark.java @@ -52,6 +52,7 @@ import org.apache.kafka.image.MetadataProvenance; import org.apache.kafka.metadata.KRaftMetadataCache; import org.apache.kafka.metadata.MockConfigRepository; +import org.apache.kafka.network.Request; import org.apache.kafka.network.RequestConvertToJson; import org.apache.kafka.network.metrics.RequestChannelMetrics; import org.apache.kafka.raft.KRaftConfigs; @@ -89,8 +90,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; -import scala.Option; - @State(Scope.Benchmark) @Fork(value = 1) @Warmup(iterations = 5) @@ -128,7 +127,7 @@ public class KRaftMetadataRequestBenchmark { @Param({"10", "20", "50"}) private int partitionCount; private KafkaApis kafkaApis; - private RequestChannel.Request allTopicMetadataRequest; + private Request allTopicMetadataRequest; @Setup(Level.Trial) public void setup() { @@ -221,7 +220,7 @@ public void tearDown() { metrics.close(); } - private RequestChannel.Request buildAllTopicMetadataRequest() { + private Request buildAllTopicMetadataRequest() { MetadataRequest metadataRequest = MetadataRequest.Builder.allTopics().build(); RequestHeader header = new RequestHeader(metadataRequest.apiKey(), metadataRequest.version(), "", 0); ByteBuffer bodyBuffer = metadataRequest.serialize().buffer(); @@ -229,7 +228,7 @@ private RequestChannel.Request buildAllTopicMetadataRequest() { RequestContext context = new RequestContext(header, "1", null, principal, ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT), SecurityProtocol.PLAINTEXT, ClientInformation.EMPTY, false); - return new RequestChannel.Request(1, context, 0, MemoryPool.NONE, bodyBuffer, requestChannelMetrics, Option.empty()); + return new Request(1, context, 0, MemoryPool.NONE, bodyBuffer, requestChannelMetrics); } @Benchmark @@ -239,8 +238,6 @@ public void testMetadataRequestForAllTopics() { @Benchmark public String testRequestToJson() { - Option option = allTopicMetadataRequest.requestLog(); - Optional optional = option.isDefined() ? Optional.of(option.get()) : Optional.empty(); - return RequestConvertToJson.requestDesc(allTopicMetadataRequest.header(), optional, allTopicMetadataRequest.isForwarded()).toString(); + return RequestConvertToJson.requestDesc(allTopicMetadataRequest.header(), allTopicMetadataRequest.requestLog(), allTopicMetadataRequest.isForwarded()).toString(); } } diff --git a/server/src/main/java/org/apache/kafka/network/BaseRequest.java b/server/src/main/java/org/apache/kafka/network/BaseRequest.java new file mode 100644 index 0000000000000..036cc9f50052a --- /dev/null +++ b/server/src/main/java/org/apache/kafka/network/BaseRequest.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.network; + +public sealed interface BaseRequest permits CallbackRequest, Request, ShutdownRequest, WakeupRequest { +} diff --git a/server/src/main/java/org/apache/kafka/network/CallbackRequest.java b/server/src/main/java/org/apache/kafka/network/CallbackRequest.java new file mode 100644 index 0000000000000..732b4b4a3c116 --- /dev/null +++ b/server/src/main/java/org/apache/kafka/network/CallbackRequest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.network; + +import org.apache.kafka.server.common.RequestLocal; + +import java.util.function.Consumer; + +public record CallbackRequest(Consumer fun, Request originalRequest) implements BaseRequest { + + @Override + public String toString() { + return "CallbackRequest(" + fun + ", " + originalRequest + ")"; + } +} diff --git a/server/src/main/java/org/apache/kafka/network/Request.java b/server/src/main/java/org/apache/kafka/network/Request.java new file mode 100644 index 0000000000000..3b3104eb667ae --- /dev/null +++ b/server/src/main/java/org/apache/kafka/network/Request.java @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.network; + +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.memory.MemoryPool; +import org.apache.kafka.common.message.EnvelopeResponseData; +import org.apache.kafka.common.network.Send; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.AddPartitionsToTxnRequest; +import org.apache.kafka.common.requests.AlterConfigsRequest; +import org.apache.kafka.common.requests.DescribeQuorumResponse; +import org.apache.kafka.common.requests.EnvelopeResponse; +import org.apache.kafka.common.requests.FetchRequest; +import org.apache.kafka.common.requests.IncrementalAlterConfigsRequest; +import org.apache.kafka.common.requests.RequestAndSize; +import org.apache.kafka.common.requests.RequestContext; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.network.metrics.RequestChannelMetrics; +import org.apache.kafka.network.metrics.RequestMetrics; +import org.apache.kafka.server.config.AbstractKafkaConfig; + +import com.fasterxml.jackson.databind.JsonNode; +import com.yammer.metrics.core.Meter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.TimeUnit; +import java.util.function.LongConsumer; + +public final class Request implements BaseRequest { + private static final Logger LOG = LoggerFactory.getLogger(Request.class); + private static final Logger REQUEST_LOGGER = LoggerFactory.getLogger("kafka.request.logger"); + + private final int processor; + private final RequestContext context; + private final long startTimeNanos; + private final MemoryPool memoryPool; + private final Session session; + + private volatile ByteBuffer buffer; + private final RequestChannelMetrics metrics; + private final Optional envelope; + private final RequestAndSize bodyAndSize; + private final JsonNode requestLog; + + // These need to be volatile because the readers are in the network thread and the writers are in the request + // handler threads or the purgatory threads + private volatile long requestDequeueTimeNanos = -1L; + private volatile long apiLocalCompleteTimeNanos = -1L; + private volatile long responseCompleteTimeNanos = -1L; + private volatile long responseDequeueTimeNanos = -1L; + private volatile long messageConversionsTimeNanos = 0L; + private volatile long apiThrottleTimeMs = 0L; + private volatile long temporaryMemoryBytes = 0L; + + private volatile OptionalLong callbackRequestDequeueTimeNanos = OptionalLong.empty(); + private volatile OptionalLong callbackRequestCompleteTimeNanos = OptionalLong.empty(); + private volatile LongConsumer recordNetworkThreadTimeCallback = __ -> { }; + + private static boolean isRequestLoggingEnabled(RequestHeader header) { + return REQUEST_LOGGER.isDebugEnabled() + || (REQUEST_LOGGER.isInfoEnabled() && header.isApiVersionDeprecated()); + } + + public Request(int processor, RequestContext context, long startTimeNanos, + MemoryPool memoryPool, ByteBuffer buffer, + RequestChannelMetrics metrics, Optional envelope) { + this.processor = processor; + this.context = context; + this.startTimeNanos = startTimeNanos; + this.memoryPool = memoryPool; + this.buffer = buffer; + this.metrics = metrics; + this.envelope = envelope; + + this.session = new Session(context.principal, context.clientAddress); + this.bodyAndSize = context.parseRequest(buffer); + + // This is constructed on creation of a Request so that the JSON representation is computed before the request is + // processed by the api layer. Otherwise, a ProduceRequest can occur without its data (ie. it goes into purgatory). + this.requestLog = isRequestLoggingEnabled(context.header) + ? RequestConvertToJson.request(loggableRequest()) + : null; + + // Most request types are parsed entirely into objects at this point. For those we can release the underlying + // buffer. Some (like produce, or any time the schema contains fields of types BYTES or NULLABLE_BYTES) retain + // a reference to the buffer. For those requests we cannot release the buffer early, but only when request + // processing is done. + if (!header().apiKey().requiresDelayedAllocation) { + releaseBuffer(); + } + + if (LOG.isTraceEnabled()) { + LOG.trace("Processor {} received request: {}", processor, requestDesc(true)); + } + } + + public Request(int processor, RequestContext context, long startTimeNanos, + MemoryPool memoryPool, ByteBuffer buffer, + RequestChannelMetrics metrics) { + this(processor, context, startTimeNanos, memoryPool, buffer, metrics, Optional.empty()); + } + + public int processor() { + return processor; + } + + public RequestContext context() { + return context; + } + + public long startTimeNanos() { + return startTimeNanos; + } + + public MemoryPool memoryPool() { + return memoryPool; + } + + public Session session() { + return session; + } + + public long requestDequeueTimeNanos() { + return requestDequeueTimeNanos; + } + + public void requestDequeueTimeNanos(long requestDequeueTimeNanos) { + this.requestDequeueTimeNanos = requestDequeueTimeNanos; + } + + public long apiLocalCompleteTimeNanos() { + return apiLocalCompleteTimeNanos; + } + + public void apiLocalCompleteTimeNanos(long apiLocalCompleteTimeNanos) { + this.apiLocalCompleteTimeNanos = apiLocalCompleteTimeNanos; + } + + public void responseCompleteTimeNanos(long responseCompleteTimeNanos) { + this.responseCompleteTimeNanos = responseCompleteTimeNanos; + } + + public void responseDequeueTimeNanos(long responseDequeueTimeNanos) { + this.responseDequeueTimeNanos = responseDequeueTimeNanos; + } + + public long messageConversionsTimeNanos() { + return messageConversionsTimeNanos; + } + + public void messageConversionsTimeNanos(long messageConversionsTimeNanos) { + this.messageConversionsTimeNanos = messageConversionsTimeNanos; + } + + public void apiThrottleTimeMs(long apiThrottleTimeMs) { + this.apiThrottleTimeMs = apiThrottleTimeMs; + } + + public long temporaryMemoryBytes() { + return temporaryMemoryBytes; + } + + public void temporaryMemoryBytes(long temporaryMemoryBytes) { + this.temporaryMemoryBytes = temporaryMemoryBytes; + } + + public OptionalLong callbackRequestDequeueTimeNanos() { + return callbackRequestDequeueTimeNanos; + } + + public void callbackRequestDequeueTimeNanos(OptionalLong callbackRequestDequeueTimeNanos) { + this.callbackRequestDequeueTimeNanos = callbackRequestDequeueTimeNanos; + } + + public OptionalLong callbackRequestCompleteTimeNanos() { + return callbackRequestCompleteTimeNanos; + } + + public void callbackRequestCompleteTimeNanos(OptionalLong callbackRequestCompleteTimeNanos) { + this.callbackRequestCompleteTimeNanos = callbackRequestCompleteTimeNanos; + } + + public RequestHeader header() { + return context.header; + } + + private int sizeOfBodyInBytes() { + return bodyAndSize.size; + } + + public int sizeInBytes() { + return header().size() + sizeOfBodyInBytes(); + } + + public boolean isForwarded() { + return envelope.isPresent(); + } + + public Optional envelope() { + return envelope; + } + + public Optional requestLog() { + return Optional.ofNullable(requestLog); + } + + public ByteBuffer buffer() { + return buffer; + } + + private boolean shouldReturnNotController(AbstractResponse response) { + if (response instanceof DescribeQuorumResponse) { + return response.errorCounts().containsKey(Errors.NOT_LEADER_OR_FOLLOWER); + } + return response.errorCounts().containsKey(Errors.NOT_CONTROLLER); + } + + public Send buildResponseSend(AbstractResponse abstractResponse) { + if (envelope.isPresent()) { + EnvelopeResponse envelopeResponse; + if (shouldReturnNotController(abstractResponse)) { + envelopeResponse = new EnvelopeResponse(new EnvelopeResponseData() + .setErrorCode(Errors.NOT_CONTROLLER.code())); + } else { + ByteBuffer responseBytes = context.buildResponseEnvelopePayload(abstractResponse); + envelopeResponse = new EnvelopeResponse(responseBytes, Errors.NONE); + } + return envelope.get().context.buildResponseSend(envelopeResponse); + } + return context.buildResponseSend(abstractResponse); + } + + public Optional responseNode(AbstractResponse response) { + if (isRequestLoggingEnabled(context.header)) { + return Optional.of(RequestConvertToJson.response(response, context.apiVersion())); + } + return Optional.empty(); + } + + public RequestHeader headerForLoggingOrThrottling() { + if (envelope.isPresent()) { + return envelope.get().context.header; + } + return context.header; + } + + public String requestDesc(boolean details) { + String forwardDescription = envelope.map(request -> "Forwarded request: " + request.context + " ").orElse(""); + return forwardDescription + header() + " -- " + loggableRequest().toString(details); + } + + public T body(Class clazz) { + if (clazz.isInstance(bodyAndSize.request)) { + return clazz.cast(bodyAndSize.request); + } + throw new ClassCastException("Expected request with type " + clazz.getName() + + ", but found " + bodyAndSize.request.getClass().getName()); + } + + public AbstractRequest loggableRequest() { + if (bodyAndSize.request instanceof AlterConfigsRequest alterConfigs) { + var newData = alterConfigs.data().duplicate(); + newData.resources().forEach(resource -> { + ConfigResource.Type resourceType = ConfigResource.Type.forId(resource.resourceType()); + resource.configs().forEach(config -> + config.setValue(AbstractKafkaConfig.loggableValue(resourceType, config.name(), config.value())) + ); + }); + return new AlterConfigsRequest(newData, alterConfigs.version()); + } else if (bodyAndSize.request instanceof IncrementalAlterConfigsRequest alterConfigs) { + var newData = alterConfigs.data().duplicate(); + newData.resources().forEach(resource -> { + ConfigResource.Type resourceType = ConfigResource.Type.forId(resource.resourceType()); + resource.configs().forEach(config -> + config.setValue(AbstractKafkaConfig.loggableValue(resourceType, config.name(), config.value())) + ); + }); + return new IncrementalAlterConfigsRequest.Builder(newData).build(alterConfigs.version()); + } + return bodyAndSize.request; + } + + public long requestThreadTimeNanos() { + if (apiLocalCompleteTimeNanos == -1L) { + apiLocalCompleteTimeNanos = Time.SYSTEM.nanoseconds(); + } + return Math.max(apiLocalCompleteTimeNanos - requestDequeueTimeNanos, 0L); + } + + /** + * Converts nanos to millis with micros precision as additional decimal places in the request log have low + * signal to noise ratio. When it comes to metrics, there is little difference either way as we round the value + * to the nearest long. + */ + private static double nanosToMs(long nanos) { + long positiveNanos = Math.max(nanos, 0); + return (double) TimeUnit.NANOSECONDS.toMicros(positiveNanos) / 1000; + } + + public void updateRequestMetrics(long networkThreadTimeNanos, Optional responseLog) { + long endTimeNanos = Time.SYSTEM.nanoseconds(); + + double requestQueueTimeMs = nanosToMs(requestDequeueTimeNanos - startTimeNanos); + long cbCompleteNanos = callbackRequestCompleteTimeNanos.orElse(0L); + long cbDequeueNanos = callbackRequestDequeueTimeNanos.orElse(0L); + long callbackRequestTimeNanos = cbCompleteNanos - cbDequeueNanos; + double apiLocalTimeMs = nanosToMs(apiLocalCompleteTimeNanos - requestDequeueTimeNanos + callbackRequestTimeNanos); + double apiRemoteTimeMs = nanosToMs(responseCompleteTimeNanos - apiLocalCompleteTimeNanos - callbackRequestTimeNanos); + double responseQueueTimeMs = nanosToMs(responseDequeueTimeNanos - responseCompleteTimeNanos); + double responseSendTimeMs = nanosToMs(endTimeNanos - responseDequeueTimeNanos); + double messageConversionsTimeMs = nanosToMs(messageConversionsTimeNanos); + double totalTimeMs = nanosToMs(endTimeNanos - startTimeNanos); + + List overrideMetricNames = switch (header().apiKey()) { + case FETCH -> { + String specifiedMetricName = body(FetchRequest.class).isFromFollower() + ? RequestMetrics.FOLLOW_FETCH_METRIC_NAME + : RequestMetrics.CONSUMER_FETCH_METRIC_NAME; + yield List.of(specifiedMetricName, ApiKeys.FETCH.name); + } + + case ADD_PARTITIONS_TO_TXN -> body(AddPartitionsToTxnRequest.class).allVerifyOnlyRequest() + ? List.of(RequestMetrics.VERIFY_PARTITIONS_IN_TXN_METRIC_NAME) + : List.of(ApiKeys.ADD_PARTITIONS_TO_TXN.name); + + case LIST_CONFIG_RESOURCES -> header().apiVersion() == 0 + ? List.of(RequestMetrics.LIST_CLIENT_METRICS_RESOURCES_METRIC_NAME, ApiKeys.LIST_CONFIG_RESOURCES.name) + : List.of(ApiKeys.LIST_CONFIG_RESOURCES.name); + + default -> List.of(header().apiKey().name); + }; + + for (String metricName : overrideMetricNames) { + RequestMetrics m = metrics.apply(metricName); + m.requestRate(header().apiVersion()).mark(); + m.deprecatedRequestRate(header().apiKey(), header().apiVersion(), context.clientInformation) + .ifPresent(Meter::mark); + m.requestQueueTimeHist.update(Math.round(requestQueueTimeMs)); + m.localTimeHist.update(Math.round(apiLocalTimeMs)); + m.remoteTimeHist.update(Math.round(apiRemoteTimeMs)); + m.throttleTimeHist.update(apiThrottleTimeMs); + m.responseQueueTimeHist.update(Math.round(responseQueueTimeMs)); + m.responseSendTimeHist.update(Math.round(responseSendTimeMs)); + m.totalTimeHist.update(Math.round(totalTimeMs)); + m.requestBytesHist.update(sizeOfBodyInBytes()); + m.messageConversionsTimeHist.ifPresent(h -> h.update(Math.round(messageConversionsTimeMs))); + m.tempMemoryBytesHist.ifPresent(h -> h.update(temporaryMemoryBytes)); + } + + // Records network handler thread usage. This is included towards the request quota for the + // user/client. Throttling is only performed when request handler thread usage + // is recorded, just before responses are queued for delivery. + // The time recorded here is the time spent on the network thread for receiving this request + // and sending the response. Note that for the first request on a connection, the time includes + // the total time spent on authentication, which may be significant for SASL/SSL. + recordNetworkThreadTimeCallback.accept(networkThreadTimeNanos); + + if (isRequestLoggingEnabled(header())) { + JsonNode desc = RequestConvertToJson.requestDescMetrics(header(), requestLog(), responseLog, + context, session, isForwarded(), + totalTimeMs, requestQueueTimeMs, apiLocalTimeMs, + apiRemoteTimeMs, apiThrottleTimeMs, responseQueueTimeMs, + responseSendTimeMs, temporaryMemoryBytes, + messageConversionsTimeMs); + String logPrefix = "Completed request:{}"; + if (header().isApiVersionDeprecated()) { + REQUEST_LOGGER.info(logPrefix, desc); + } else { + REQUEST_LOGGER.debug(logPrefix, desc); + } + } + } + + public void releaseBuffer() { + if (envelope.isPresent()) { + envelope.get().releaseBuffer(); + } else { + if (buffer != null) { + memoryPool.release(buffer); + buffer = null; + } + } + } + + public void setRecordNetworkThreadTimeCallback(LongConsumer callback) { + this.recordNetworkThreadTimeCallback = callback; + } + + @Override + public String toString() { + return "Request(processor=" + processor + + ", connectionId=" + context.connectionId + + ", session=" + session + + ", listenerName=" + context.listenerName + + ", securityProtocol=" + context.securityProtocol + + ", buffer=" + buffer + + ", envelope=" + envelope + ")"; + } +} diff --git a/server/src/main/java/org/apache/kafka/network/ShutdownRequest.java b/server/src/main/java/org/apache/kafka/network/ShutdownRequest.java new file mode 100644 index 0000000000000..bf40cca5275a8 --- /dev/null +++ b/server/src/main/java/org/apache/kafka/network/ShutdownRequest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.network; + +public enum ShutdownRequest implements BaseRequest { + INSTANCE; + + @Override + public String toString() { + return "ShutdownRequest"; + } +} diff --git a/server/src/main/java/org/apache/kafka/network/WakeupRequest.java b/server/src/main/java/org/apache/kafka/network/WakeupRequest.java new file mode 100644 index 0000000000000..43268868cd9fd --- /dev/null +++ b/server/src/main/java/org/apache/kafka/network/WakeupRequest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.network; + +public enum WakeupRequest implements BaseRequest { + INSTANCE; + + @Override + public String toString() { + return "WakeupRequest"; + } +}