KeyArgo
/
private-gpt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: Adding an LLM param to fix broken generator from llamacpp (#1519)

This commit is contained in:
Naveen Kannan 2024-01-17 12:10:45 -05:00 committed by GitHub
parent e326126d0d
commit 869233f0e4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
private_gpt/components/llm/llm_component.py
View File

@ -42,7 +42,7 @@ class LLMComponent:
context_window=settings.llm.context_window, context_window=settings.llm.context_window,
generate_kwargs={}, generate_kwargs={},
# All to GPU # All to GPU
model_kwargs={"n_gpu_layers": -1}, model_kwargs={"n_gpu_layers": -1, "offload_kqv": True},
# transform inputs into Llama2 format # transform inputs into Llama2 format
messages_to_prompt=prompt_style.messages_to_prompt, messages_to_prompt=prompt_style.messages_to_prompt,
completion_to_prompt=prompt_style.completion_to_prompt, completion_to_prompt=prompt_style.completion_to_prompt,