Refactor Benchmark Report Generation and Update Backend Configurations (#33)

## Summary
This PR refactors the benchmark report generation process and updates
backend configurations for better performance and clarity. The main
changes include restructuring the entry point for benchmark generation,
consolidating backend configurations, and improving error handling for
better user experience.

## Details
- Introduced `generate_benchmark_report_cli` as the main CLI entry point
for benchmark generation.
- Refactored `generate_benchmark_report` to handle various input
configurations and streamline the execution process.
- Removed unused parameters (`host`, `port`) and consolidated backend
URL configurations into a single `target` parameter.
- Updated `OpenAIBackend` initialization to eliminate ambiguity around
base URL settings.
- Enhanced error handling and logging, ensuring informative messages for
missing or incorrect configurations.
- Modified `EmulatedRequestGenerator` and `FileRequestGenerator` to
handle optional configuration inputs.
- Removed obsolete CLI test files and restructured e2e tests to align
with the new CLI entry point.
- Added comprehensive unit tests for the main benchmark generation
functions and CLI commands.

Author: markurtz

src/guidellm/__init__.py

@@ -3,15 +3,17 @@
 evaluating and benchmarking large language models (LLMs).
 """
 
+# flake8: noqa
+
 import os
+import transformers  # type: ignore
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"  # Silence warnings for tokenizers
+transformers.logging.set_verbosity_error()  # Silence warnings for transformers
 
-import transformers  # type: ignore  # noqa: PGH003
 
 from .config import settings
 from .logger import configure_logger, logger
+from .main import generate_benchmark_report
 
-__all__ = ["configure_logger", "logger", "settings"]
-
-
-os.environ["TOKENIZERS_PARALLELISM"] = "false"  # Silence warnings for tokenizers
-transformers.logging.set_verbosity_error()  # Silence warnings for transformers
+__all__ = ["configure_logger", "logger", "settings", "generate_benchmark_report"]

src/guidellm/backend/__init__.py

@@ -1,9 +1,10 @@
-from .base import Backend, BackendEngine, GenerativeResponse
+from .base import Backend, BackendEngine, BackendEnginePublic, GenerativeResponse
 from .openai import OpenAIBackend
 
 __all__ = [
     "Backend",
     "BackendEngine",
+    "BackendEnginePublic",
     "GenerativeResponse",
     "OpenAIBackend",
 ]

src/guidellm/backend/base.py

@@ -1,16 +1,17 @@
 import functools
 from abc import ABC, abstractmethod
-from typing import AsyncGenerator, Dict, List, Literal, Optional, Type
+from typing import AsyncGenerator, Dict, List, Literal, Optional, Type, Union
 
 from loguru import logger
 from pydantic import BaseModel
 
 from guidellm.core import TextGenerationRequest, TextGenerationResult
 
-__all__ = ["Backend", "BackendEngine", "GenerativeResponse"]
+__all__ = ["Backend", "BackendEngine", "BackendEnginePublic", "GenerativeResponse"]
 
 
-BackendEngine = Literal["test", "openai_server"]
+BackendEnginePublic = Literal["openai_server"]
+BackendEngine = Union[BackendEnginePublic, Literal["test"]]
 
 
 class GenerativeResponse(BaseModel):
@@ -87,7 +88,6 @@ def create(cls, backend_type: BackendEngine, **kwargs) -> "Backend":
         :param backend_type: The type of backend to create.
         :type backend_type: BackendEngine
         :param kwargs: Additional arguments for backend initialization.
-        :type kwargs: dict
         :return: An instance of a subclass of Backend.
         :rtype: Backend
         :raises ValueError: If the backend type is not registered.

src/guidellm/backend/openai.py

@@ -23,10 +23,6 @@ class OpenAIBackend(Backend):
     :type openai_api_key: Optional[str]
     :param target: The target URL string for the OpenAI server.
     :type target: Optional[str]
-    :param host: Optional host for the OpenAI server.
-    :type host: Optional[str]
-    :param port: Optional port for the OpenAI server.
-    :type port: Optional[int]
     :param model: The OpenAI model to use, defaults to the first available model.
     :type model: Optional[str]
     :param request_args: Additional arguments for the OpenAI request.
@@ -37,8 +33,6 @@ def __init__(
         self,
         openai_api_key: Optional[str] = None,
         target: Optional[str] = None,
-        host: Optional[str] = None,
-        port: Optional[int] = None,
         model: Optional[str] = None,
         **request_args,
     ):
@@ -54,16 +48,12 @@ def __init__(
             logger.error("{}", err)
             raise err
 
-        if target:
-            base_url = target
-        elif host and port:
-            base_url = f"{host}:{port}/v1"
-        elif settings.openai.base_url:
-            base_url = settings.openai.base_url
-        else:
+        base_url = target or settings.openai.base_url
+
+        if not base_url:
             err = ValueError(
                 "`GUIDELLM__OPENAI__BASE_URL` environment variable or "
-                "--target CLI parameter must be specified for the OpenAI backend."
+                "target parameter must be specified for the OpenAI backend."
             )
             logger.error("{}", err)
             raise err

src/guidellm/config.py

@@ -102,7 +102,7 @@ class OpenAISettings(BaseModel):
 
     # OpenAI-compatible server URL
     # NOTE: The default value is default address of llama.cpp web server
-    base_url: str = "http://localhost:8080"
+    base_url: str = "http://localhost:8000/v1"
 
     max_gen_tokens: int = 4096