Update app.py
Browse files
app.py
CHANGED
|
@@ -8,7 +8,7 @@ from llama_cpp import Llama
|
|
| 8 |
|
| 9 |
# Set the model repo + filename (change filename if needed)
|
| 10 |
MODEL_REPO = "mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated-GGUF"
|
| 11 |
-
MODEL_FILE = "
|
| 12 |
|
| 13 |
print("Downloading model from HF Hub...")
|
| 14 |
model_path = hf_hub_download(
|
|
@@ -23,7 +23,7 @@ print(f"Model downloaded to: {model_path}")
|
|
| 23 |
|
| 24 |
llm = Llama(
|
| 25 |
model_path=model_path,
|
| 26 |
-
n_gpu_layers
|
| 27 |
n_ctx=4096,
|
| 28 |
chat_format="llama-3",
|
| 29 |
)
|
|
|
|
| 8 |
|
| 9 |
# Set the model repo + filename (change filename if needed)
|
| 10 |
MODEL_REPO = "mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated-GGUF"
|
| 11 |
+
MODEL_FILE = "meta-llama-3.1-8b-instruct-abliterated.Q4_K_M.gguf"
|
| 12 |
|
| 13 |
print("Downloading model from HF Hub...")
|
| 14 |
model_path = hf_hub_download(
|
|
|
|
| 23 |
|
| 24 |
llm = Llama(
|
| 25 |
model_path=model_path,
|
| 26 |
+
n_gpu_layers=0, # GPU acceleration if available
|
| 27 |
n_ctx=4096,
|
| 28 |
chat_format="llama-3",
|
| 29 |
)
|