Update app.py

app.py

@@ -1,10 +1,10 @@
 import gradio as gr
-import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer
 from threading import Thread
+import bitsandbytes
 
 tokenizer = AutoTokenizer.from_pretrained("./model/")
-model = AutoModelForCausalLM.from_pretrained("./model/", torch_dtype=torch.float16)
+model = AutoModelForCausalLM.from_pretrained("./model/", device_map="auto", load_in_4bit=True)
 model = model.to('cuda:0')
 
 class StopOnTokens(StoppingCriteria):