From 0509f4f0f28a38781a7437bdb5d24f30b301eab8 Mon Sep 17 00:00:00 2001 From: Shourya Shashank Date: Tue, 22 Oct 2024 22:20:58 +0530 Subject: [PATCH 1/3] Update cli.py --- app/predacons_cli/src/cli.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/app/predacons_cli/src/cli.py b/app/predacons_cli/src/cli.py index a83915a..499934e 100644 --- a/app/predacons_cli/src/cli.py +++ b/app/predacons_cli/src/cli.py @@ -97,6 +97,10 @@ def launch(self,logs=False): for i in range(1, 0, -1): time.sleep(1) os.system('clear') # Clear the screen + personality = None + if config.get("personality", None): + print("[yellow]Personality detected! Loading the quirks and traits... Brace yourself![/yellow]") + personality = "\n Make sure to respond back as a " + config["personality"] print("[i]Welcome to the Predacons CLI![/i] [green]Model: [orange1]"+config["model_path"]+"[/orange1] loaded successfully![/green]") print("[yellow]You can start chatting with Predacons now.Type 'clear' to clear history, Type 'exit' to quit, Type 'help' for more options, Type 'update' to update the load documents[/yellow]") @@ -143,7 +147,7 @@ def launch(self,logs=False): Answer the question based on the above context: {question} """ - + PROMPT_TEMPLATE = PROMPT_TEMPLATE + personality if personality else "" user_input = PROMPT_TEMPLATE.format(db_context=context_text,web_context=web_text, question=user_input) PROMPT_TEMPLATE = """ @@ -269,7 +273,8 @@ def create_config_file(self): "vector_db_path": None, "document_path": None, "embedding_model" : None, - "scrap_web": False + "scrap_web": False, + "personality": None } # If no config file is found, use the default configuration @@ -330,6 +335,7 @@ def create_config_file(self): document_path = Prompt.ask("Enter the document path", default=config["document_path"]) embedding_model = Prompt.ask("Enter the embedding model", default=config["embedding_model"]) scrap_web = Prompt.ask("Scrap the web for data? (true/false)", default=str(config["scrap_web"])) + personality = Prompt.ask("Enter the personality", default=config["personality"]) config_data = { @@ -350,7 +356,8 @@ def create_config_file(self): "vector_db_path": vector_db_path if vector_db_path else None, "document_path": document_path if document_path else None, "embedding_model": embedding_model if embedding_model else None, - "scrap_web": scrap_web.lower() == 'true' + "scrap_web": scrap_web.lower() == 'true', + "personality": personality if personality else None } with open(file_path, 'w') as f: @@ -377,7 +384,7 @@ def generate_response(self, chat, model, tokenizer, config): ) return response -# cli = Cli() -# cli.launch() +cli = Cli() +cli.launch() From 7c037b9a347c7315be9c21bb93162b3dddaf2c05 Mon Sep 17 00:00:00 2001 From: Shourya Shashank Date: Sat, 2 Nov 2024 20:40:30 +0530 Subject: [PATCH 2/3] Update cli.py --- app/predacons_cli/src/cli.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/app/predacons_cli/src/cli.py b/app/predacons_cli/src/cli.py index 499934e..5d50aea 100644 --- a/app/predacons_cli/src/cli.py +++ b/app/predacons_cli/src/cli.py @@ -8,6 +8,7 @@ from rich.table import Table from rich.markdown import Markdown from rich.console import Console +from transformers import TextStreamer try: from .rag import VectorStore from .rag import WebScraper @@ -177,15 +178,24 @@ def launch(self,logs=False): user_input = PROMPT_TEMPLATE.format(context=web_text, question=user_input) user_body = {"role": "user", "content": user_input} chat.append(user_body) - response = Cli.generate_response(self, chat, model, tokenizer, config) + thread,streamer = Cli.generate_response(self, chat, model, tokenizer, config) + thread.start() + try: + response = "" + for new_text in streamer: + response = response + new_text + print(new_text, end="") + print("\n") + finally: + thread.join() response_body = {"role": "assistant", "content": response} chat.append(response_body) if config["print_as_markdown"]: markdown = Markdown(response) print("[orange1]Predacons: [/orange1]") console.print(markdown) - else: - console.print("[orange1]Predacons: [/orange1] [sky_blue1]" + response+"[/sky_blue1]") + # else: + # console.print("[orange1]Predacons: [/orange1] [sky_blue1]" + response+"[/sky_blue1]") def load_model(self, model_path,trust_remote_code=False,use_fast_generation=False, draft_model_name=None,gguf_file=None,auto_quantize=None): @@ -373,6 +383,19 @@ def load_config_file(self): return config def generate_response(self, chat, model, tokenizer, config): + thread,streamer = self.predacons.chat_generate(model = model, + sequence = chat, + max_length = config["max_length"], + tokenizer = tokenizer, + trust_remote_code = config["trust_remote_code"], + do_sample=True, + temperature = config["temperature"], + dont_print_output = True, + stream = True + ) + return thread,streamer + + def generate_response2(self, chat, model, tokenizer, config): response = self.predacons.chat_generate(model = model, sequence = chat, max_length = config["max_length"], From f582a12e47f4a16bacf15f95c7ada0681f1e3670 Mon Sep 17 00:00:00 2001 From: Shourya Shashank Date: Sat, 2 Nov 2024 22:48:40 +0530 Subject: [PATCH 3/3] Update cli.py --- app/predacons_cli/src/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/predacons_cli/src/cli.py b/app/predacons_cli/src/cli.py index 5d50aea..74d2cc4 100644 --- a/app/predacons_cli/src/cli.py +++ b/app/predacons_cli/src/cli.py @@ -180,11 +180,12 @@ def launch(self,logs=False): chat.append(user_body) thread,streamer = Cli.generate_response(self, chat, model, tokenizer, config) thread.start() + print("[orange1]Predacons: [/orange1]", end="") try: response = "" for new_text in streamer: response = response + new_text - print(new_text, end="") + print("[sky_blue1]"+ new_text + "[/sky_blue1]", end="") print("\n") finally: thread.join()