fix : server crash with ollama

2025-03-29 18:07:38 +01:00
parent d8ded2d456
commit e0eee90202
4 changed files with 25 additions and 12 deletions
@@ -0,0 +1,37 @@
+
+from .generator import GeneratorLLM
+from llama_cpp import Llama
+
+class LlamacppLLM(GeneratorLLM):
+
+    def __init__(self):
+        """
+        Handle generation using llama.cpp
+        """
+        super().__init__()
+        self.llm = None
+    
+    def generate(self, history):
+        if self.llm is None:
+            self.llm = Llama.from_pretrained(
+                repo_id=self.model,
+                filename="*Q8_0.gguf",
+                verbose=True
+            )
+            return
+        self.logger.info(f"Using {self.model} for generation with Llama.cpp")
+        try:
+            with self.state.lock:
+                self.state.is_generating = True
+                self.state.last_complete_sentence = ""
+                self.state.current_buffer = ""
+            output = self.llm.create_chat_completion(
+                  messages = history
+            )
+            with self.state.lock:
+                self.state.current_buffer = output
+        except Exception as e:
+            self.logger.error(f"Error: {e}")
+        finally:
+            with self.state.lock:
+                self.state.is_generating = False