fix : server problem with llamacpp
This commit is contained in:
@@ -9,13 +9,15 @@ class LlamacppLLM(GeneratorLLM):
|
||||
Handle generation using llama.cpp
|
||||
"""
|
||||
super().__init__()
|
||||
self.llm = Llama.from_pretrained(
|
||||
repo_id=self.model,
|
||||
filename="*q8_0.gguf",
|
||||
verbose=True
|
||||
)
|
||||
self.llm = None
|
||||
|
||||
def generate(self, history):
|
||||
if self.model is None:
|
||||
self.llm = Llama.from_pretrained(
|
||||
repo_id=self.model,
|
||||
filename="*q8_0.gguf",
|
||||
verbose=True
|
||||
)
|
||||
self.logger.info(f"Using {self.model} for generation with Llama.cpp")
|
||||
self.llm.create_chat_completion(
|
||||
messages = history
|
||||
|
||||
Reference in New Issue
Block a user