diff --git a/app/predacons/src/generate.py b/app/predacons/src/generate.py index d5ddf27..251dc4b 100644 --- a/app/predacons/src/generate.py +++ b/app/predacons/src/generate.py @@ -45,13 +45,47 @@ def __load_model(model_path, trust_remote_code=False,gguf_file=None,auto_quantiz def __load_tokenizer(tokenizer_path,gguf_file=None): + """ + Loads a tokenizer from the specified path. + + Args: + tokenizer_path: Path to the pretrained tokenizer. + gguf_file: Optional file for GGUF format support. + + Returns: + The loaded tokenizer instance. + """ tokenizer = AutoTokenizer.from_pretrained(tokenizer_path,gguf_file=gguf_file) return tokenizer def __load_processor(tokenizer_path,use_fast=False,gguf_file=None): + """ + Loads a processor from the specified path. + + Args: + tokenizer_path: Path to the pretrained processor or model directory. + use_fast: Whether to use the fast implementation if available. + gguf_file: Optional GGUF file for processor configuration. + + Returns: + An instance of AutoProcessor loaded from the given path. + """ processor = AutoProcessor.from_pretrained(tokenizer_path, use_fast=use_fast, gguf_file=gguf_file) return processor def __generate_output(model_path, sequence, max_length,trust_remote_code=False,gguf_file=None,auto_quantize=None): + """ + Generates output token IDs from a pretrained model given an input sequence. + + Loads the specified model and tokenizer, encodes the input sequence, and generates output tokens using sampling with top-k and top-p settings. + + Args: + model_path: Path to the pretrained model. + sequence: Input text sequence to generate output from. + max_length: Maximum length of the generated output. + + Returns: + A tuple containing the generated output token IDs and the tokenizer used. + """ model = Generate.__load_model(model_path,trust_remote_code=trust_remote_code,gguf_file=gguf_file,auto_quantize=auto_quantize) tokenizer = Generate.__load_tokenizer(model_path,gguf_file=gguf_file) ids = tokenizer.encode(f'{sequence}', return_tensors='pt') @@ -175,6 +209,14 @@ def __generate_chat_output_from_model(model, tokenizer, sequence, max_length,tem return inputs,final_outputs,tokenizer def __generate_chat_output_from_model_stream(model, tokenizer, sequence, max_length,temperature=0.1,trust_remote_code=False): + """ + Streams chat-style text generation from a model using a tokenizer and input sequence. + + Formats the input sequence as a chat prompt using a chat template, tokenizes it, and initiates streaming generation in a separate thread. Returns the thread and a streamer for consuming generated text in real time. + + Raises: + RuntimeError: If streaming generation setup fails. + """ try: # ids = tokenizer.encode(f'{sequence}', return_tensors='pt') if tokenizer.chat_template is None: @@ -195,6 +237,11 @@ def __generate_chat_output_from_model_stream(model, tokenizer, sequence, max_len raise RuntimeError(f"Failed to setup streaming generation: {str(e)}") def __generate_output_with_processor(model, processor, messages, max_length, temperature=0.1): + """ + Generates model outputs from chat messages using a processor for input preparation. + + If the processor lacks a chat template, a default template is applied. The messages are formatted and tokenized using the processor, moved to the model's device, and passed to the model for generation without sampling. Returns the prepared inputs, generated outputs, and the processor. + """ try: if processor.chat_template is None: print("Warning: Chat template not found in processor. Applying default chat template") @@ -210,6 +257,14 @@ def __generate_output_with_processor(model, processor, messages, max_length, tem raise RuntimeError(f"Failed to generate output with processor: {str(e)}") def __generate_output_with_processor_stream(model, processor, messages, max_length, temperature=0.1): + """ + Performs streaming text generation using a processor and a language model. + + Applies a chat template to the input messages, tokenizes them, and initiates streaming generation in a separate thread. Returns the thread and a streamer for consuming generated text in real time. + + Raises: + RuntimeError: If streaming generation setup fails. + """ try: if processor.chat_template is None: print("Warning: Chat template not found in processor. Applying default chat template") @@ -229,6 +284,14 @@ def __generate_output_with_processor_stream(model, processor, messages, max_leng except Exception as e: raise RuntimeError(f"Failed to generate output with processor stream: {str(e)}") def generate_output(model_path, sequence, max_length,trust_remote_code=False,gguf_file=None,auto_quantize=None): + """ + Generates output token IDs from a language model given an input sequence. + + Loads the specified model and tokenizer, encodes the input sequence, and generates output tokens using sampling with a maximum output length. + + Returns: + A tuple containing the generated token IDs and the tokenizer instance. + """ return Generate.__generate_output(model_path, sequence, max_length,trust_remote_code=trust_remote_code,gguf_file=gguf_file,auto_quantize=auto_quantize) def generate_output_stream(model_path, sequence, max_length,trust_remote_code=False,gguf_file=None,auto_quantize=None): @@ -238,12 +301,45 @@ def generate_text(model_path, sequence, max_length,trust_remote_code=False,gguf_ return Generate.__generate_text(model_path, sequence, max_length,trust_remote_code=trust_remote_code,gguf_file=gguf_file) def load_tokenizer(tokenizer_path,gguf_file=None): + """ + Loads and returns a tokenizer from the specified path. + + Args: + tokenizer_path: Path to the tokenizer directory or file. + gguf_file: Optional file for GGUF format support. + + Returns: + The loaded tokenizer instance. + """ return Generate.__load_tokenizer(tokenizer_path,gguf_file=gguf_file) def load_processor(tokenizer_path,use_fast=False,gguf_file=None): + """ + Loads and returns a processor from the specified path. + + Args: + tokenizer_path: Path to the processor or tokenizer directory. + use_fast: Whether to use the fast implementation, if available. + gguf_file: Optional path to a GGUF file for processor configuration. + + Returns: + An instance of the loaded processor. + """ return Generate.__load_processor(tokenizer_path,use_fast=use_fast,gguf_file=gguf_file) def load_model(model_path,trust_remote_code=False,gguf_file = None,auto_quantize=None): + """ + Loads a pretrained model from the specified path with optional quantization and trust settings. + + Args: + model_path: Path to the pretrained model directory or file. + trust_remote_code: Whether to allow execution of custom code from the model repository. + gguf_file: Optional file for GGUF format models. + auto_quantize: Optional quantization mode ("4bit", "8bit", "high", "low"). + + Returns: + The loaded model instance. + """ return Generate.__load_model(model_path,trust_remote_code=trust_remote_code,gguf_file=gguf_file,auto_quantize=auto_quantize) def generate_output_from_model(model, tokenizer, sequence, max_length,trust_remote_code=False): @@ -262,10 +358,33 @@ def generate_chat_output_from_model(model, tokenizer, sequence, max_length,tempe return Generate.__generate_chat_output_from_model(model, tokenizer, sequence, max_length,temperature=temperature,trust_remote_code=trust_remote_code) def generate_chat_output_from_model_stream(model, tokenizer, sequence, max_length,temperature=0.1,trust_remote_code=False): + """ + Streams chat-style text generation output from a pre-loaded model and tokenizer. + + Returns: + A tuple containing the thread handling generation and a streamer for iterating over generated text. + """ return Generate.__generate_chat_output_from_model_stream(model, tokenizer, sequence, max_length,temperature=temperature,trust_remote_code=trust_remote_code) def generate_output_with_processor(model, processor, messages, max_length, temperature=0.1): + """ + Generates text output from a model using a processor and a list of chat messages. + + Args: + messages: A list of chat messages to be processed and used as input. + max_length: The maximum number of tokens to generate. + temperature: Sampling temperature for generation, controlling randomness. + + Returns: + A tuple containing the processed input tensors, generated output tensors, and the processor instance. + """ return Generate.__generate_output_with_processor(model, processor, messages, max_length, temperature) def generate_output_with_processor_stream(model, processor, messages, max_length, temperature=0.1): + """ + Streams generated text output from a model using a processor and chat-style messages. + + Returns: + A tuple containing the thread handling generation and a streamer for iterating over generated text. + """ return Generate.__generate_output_with_processor_stream(model, processor, messages, max_length, temperature) \ No newline at end of file diff --git a/app/predacons/src/predacons.py b/app/predacons/src/predacons.py index 9f09114..160fcf4 100644 --- a/app/predacons/src/predacons.py +++ b/app/predacons/src/predacons.py @@ -7,6 +7,11 @@ import pandas as pd def rollout(): + """ + Prints version information and a comprehensive usage guide for all available Predacons functions. + + Displays descriptions and parameter details for data loading, training, text generation, streaming, chat, model/tokenizer loading, and data preparation functions provided by the Predacons module. + """ print("Predacons rollout !!!") print("Predacons Version: v0.0.130") print("\nread_documents_from_directory -- Load data from directory") @@ -378,33 +383,19 @@ def generate_output(model_path, sequence, max_length,trust_remote_code = False,u def generate(*args, **kwargs): """ - Generates output based on the provided arguments. - + Generates text output using a model path or preloaded model, supporting chat templates, streaming, and fast generation. + + Depending on the provided arguments, this function can generate text from a model path or a preloaded model object. It supports chat-style output, streaming results, speculative decoding (fast generation), quantization, and GGUF file integration. When using a preloaded model, either a tokenizer or processor must be provided. Raises a ValueError if required arguments are missing or invalid. + Args: - *args: Variable length arguments. - **kwargs: Keyword arguments. - - Keyword Args: - model_path (str): The path to the model file. - sequence (str): The input sequence to generate output from. - max_length (int, optional): The maximum length of the generated output. Defaults to 50. - trust_remote_code (bool, optional): Whether to trust remote code. Defaults to False. - use_fast_generation (bool, optional): Whether to use fast generation. Defaults to False. - draft_model_name (str, optional): The name of the draft model. Defaults to None. - model (object): The model object. - tokenizer (object): The tokenizer object. - processor (object): The processor object. Alternative to tokenizer for model-based generation. If provided, will be used for generation instead of tokenizer. - apply_chat_template (bool, optional): Whether to apply the chat template. Defaults to False. - temperature (float, optional): The temperature parameter for controlling the randomness of the generated output. Defaults to 0.1. - gguf_file (str, optional): The path to the GGUF file. Defaults to None. - auto_quantize (str, optional): Automatically apply quantization. Accepts "4bit"/"high" for high compression or "8bit"/"low" for lower compression. Defaults to None. - stream (bool, optional): Whether to stream the output. Defaults to False. If True, thread and streamer will be returned. - + *args: Positional arguments (not used directly). + **kwargs: Keyword arguments controlling generation behavior. + Returns: - str or tuple: The generated output, or (thread, streamer) if streaming is enabled. - + The generated text output, or a (thread, streamer) tuple if streaming is enabled. + Raises: - ValueError: If the arguments are invalid. + ValueError: If required arguments are missing or invalid. """ if 'model_path' in kwargs and ('sequence' in kwargs or 'chat' in kwargs): model_path = kwargs['model_path'] @@ -488,33 +479,9 @@ def generate(*args, **kwargs): def text_generate(*args, **kwargs): """ - Generate text using the specified arguments. - - Args: - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - - Keyword Args: - model_path (str): The path to the model file. - sequence (str): The input sequence to generate output from. - max_length (int, optional): The maximum length of the generated output. Defaults to 50. - trust_remote_code (bool, optional): Whether to trust remote code. Defaults to False. - use_fast_generation (bool, optional): Whether to use fast generation. Defaults to False. - draft_model_name (str, optional): The name of the draft model. Defaults to None. - model (object): The model object. - tokenizer (object): The tokenizer object. - processor (object): The processor object. Alternative to tokenizer for model-based generation. If provided, will be used for generation instead of tokenizer. - apply_chat_template (bool, optional): Whether to apply the chat template. Defaults to False. - temperature (float, optional): The temperature parameter for controlling the randomness of the generated output. Defaults to 0.1. - gguf_file (str, optional): The path to the GGUF file. Defaults to None. - auto_quantize (str, optional): Automatically apply quantization. Accepts "4bit"/"high" for high compression or "8bit"/"low" for lower compression. Defaults to None. - stream (bool, optional): Whether to stream the output. Defaults to False. If True, thread and streamer will be returned. - Returns: - str: The generated text. - or - thread: The thread object. - streamer: The streamer object. - + Generates text using the provided model, tokenizer, or processor, and prints the decoded output if possible. + + Supports both standard and streaming generation modes. If streaming is enabled, returns the streaming thread and streamer objects; otherwise, prints and returns the decoded text when possible, or the raw output. """ stream = kwargs.get('stream',False) if stream: @@ -539,7 +506,16 @@ def text_generate(*args, **kwargs): return output return result def _handle_stream(thread, streamer): - """Internal utility to handle streaming output.""" + """ + Starts a streaming thread and prints streamed text output incrementally. + + Args: + thread: The thread responsible for generating streamed output. + streamer: An iterable yielding chunks of text as they are generated. + + Returns: + The full concatenated output text after streaming is complete. + """ thread.start() try: out = "" @@ -551,33 +527,17 @@ def _handle_stream(thread, streamer): thread.join() def text_stream(*args, **kwargs): """ - stream text using the specified arguments. - + Streams generated text output using the provided arguments. + + Enables streaming mode for text generation and returns either the streaming objects or prints the streamed output incrementally, depending on the `return_streamer` flag. + Args: - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - - Keyword Args: - model_path (str): The path to the model file. - sequence (str): The input sequence to generate output from. - max_length (int, optional): The maximum length of the generated output. Defaults to 50. - trust_remote_code (bool, optional): Whether to trust remote code. Defaults to False. - use_fast_generation (bool, optional): Whether to use fast generation. Defaults to False. - draft_model_name (str, optional): The name of the draft model. Defaults to None. - model (object): The model object. - tokenizer (object): The tokenizer object. - processor (object): The processor object. Alternative to tokenizer for model-based generation. If provided, will be used for generation instead of tokenizer. - apply_chat_template (bool, optional): Whether to apply the chat template. Defaults to False. - temperature (float, optional): The temperature parameter for controlling the randomness of the generated output. Defaults to 0.1. - gguf_file (str, optional): The path to the GGUF file. Defaults to None. - auto_quantize (str, optional): Automatically apply quantization. Accepts "4bit"/"high" for high compression or "8bit"/"low" for lower compression. Defaults to None. - return_streamer (bool, optional): Whether to return the streamer instead of printing the text. Defaults to False. + *args: Positional arguments forwarded to the generation function. + **kwargs: Keyword arguments for generation configuration. + Returns: - str: The generated text. - or - thread: The thread object. - streamer: The streamer object. - + If `return_streamer` is True, returns the streaming thread and streamer objects. + Otherwise, prints the streamed output and returns the final text. """ kwargs['stream'] = True @@ -590,33 +550,16 @@ def text_stream(*args, **kwargs): def chat_generate(*args, **kwargs): """ - Generate chat using the specified arguments. - + Generates chat-style output using the specified model and input sequence. + + If streaming is enabled, returns the streaming thread and streamer objects. Otherwise, returns the generated chat output as a string, decoding it if possible. The function automatically applies a chat template to the input. + Args: - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - - Keyword Args: - model_path (str): The path to the model file. - sequence (str): The input sequence to generate output from. - dont_print_output (bool, optional): Whether to print the output. Defaults to False. - max_length (int, optional): The maximum length of the generated output. Defaults to 50. - trust_remote_code (bool, optional): Whether to trust remote code. Defaults to False. - use_fast_generation (bool, optional): Whether to use fast generation. Defaults to False. - draft_model_name (str, optional): The name of the draft model. Defaults to None. - model (object): The model object. - tokenizer (object): The tokenizer object. - processor (object): The processor object. Alternative to tokenizer for model-based generation. If provided, will be used for generation instead of tokenizer. - apply_chat_template (bool, optional): Whether to apply the chat template. Defaults to False. - temperature (float, optional): The temperature parameter for controlling the randomness of the generated output. Defaults to 0.1. - gguf_file (str, optional): The path to the GGUF file. Defaults to None. - auto_quantize (str, optional): Automatically apply quantization. Accepts "4bit"/"high" for high compression or "8bit"/"low" for lower compression. Defaults to None. + *args: Positional arguments forwarded to the underlying generation function. + **kwargs: Keyword arguments for generation configuration. + Returns: - str: The generated chat . - or - thread: The thread object. - streamer: The streamer object. - + The generated chat output as a string, or a tuple of (thread, streamer) if streaming is enabled. """ kwargs['apply_chat_template'] = True stream = kwargs.get('stream',False) @@ -636,32 +579,13 @@ def chat_generate(*args, **kwargs): return result def chat_stream(*args, **kwargs): """ - stream text using the specified arguments. - - Args: - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - - Keyword Args: - model_path (str): The path to the model file. - sequence (str): The input sequence to generate output from. - max_length (int, optional): The maximum length of the generated output. Defaults to 50. - trust_remote_code (bool, optional): Whether to trust remote code. Defaults to False. - use_fast_generation (bool, optional): Whether to use fast generation. Defaults to False. - draft_model_name (str, optional): The name of the draft model. Defaults to None. - model (object): The model object. - tokenizer (object): The tokenizer object. - processor (object): The processor object. Alternative to tokenizer for model-based generation. If provided, will be used for generation instead of tokenizer. - apply_chat_template (bool, optional): Whether to apply the chat template. Defaults to False. - temperature (float, optional): The temperature parameter for controlling the randomness of the generated output. Defaults to 0.1. - gguf_file (str, optional): The path to the GGUF file. Defaults to None. - auto_quantize (str, optional): Automatically apply quantization. Accepts "4bit"/"high" for high compression or "8bit"/"low" for lower compression. Defaults to None. - return_streamer (bool, optional): Whether to return the streamer instead of printing the text. Defaults to False. + Streams chat-style text generation using the provided arguments. + + Enables both streaming and chat template application, returning either the streaming objects or printing the streamed output incrementally. + Returns: - str: The generated text. - or - thread: The thread object. - streamer: The streamer object. + If `return_streamer` is True, returns the streaming thread and streamer objects. + Otherwise, prints the streamed output and returns the final result. """ kwargs['stream'] = True kwargs['apply_chat_template'] = True @@ -735,27 +659,28 @@ def load_model(model_path,trust_remote_code=False,use_fast_generation=False, dra def load_tokenizer(tokenizer_path,gguf_file=None): """ - Loads a tokenizer from the specified path. - + Loads a tokenizer from the given path. + Args: - tokenizer_path (str): The path to the tokenizer file. - + tokenizer_path: Path to the tokenizer file. + gguf_file: Optional GGUF file for tokenizer configuration. + Returns: - Tokenizer: The loaded tokenizer object. + The loaded tokenizer object. """ return Generate.load_tokenizer(tokenizer_path,gguf_file=gguf_file) def load_processor(processor_path,use_fast=False,gguf_file=None): """ - Loads a processor from the specified path. - + Loads a processor from a given path with optional fast processing and GGUF file support. + Args: - processor_path (str): The path to the processor file. - use_fast (bool, optional): Whether to use fast processing. Defaults to False. - gguf_file (str, optional): The path to the GGUF file. Defaults to None. - + processor_path: Path to the processor file. + use_fast: If True, enables fast processing mode. + gguf_file: Optional path to a GGUF file for processor configuration. + Returns: - processor: The loaded processor object. + The loaded processor object. """ return Generate.load_processor(processor_path,use_fast=use_fast,gguf_file=gguf_file)