File size: 1,007 Bytes
55a9aad 8ea488e 55a9aad 7edc096 55a9aad 8ea488e 55a9aad 3ec41f3 55a9aad 78c3718 55a9aad |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 |
import gradio as gr
from huggingface_hub import InferenceClient
def respond(
message,
history: list[dict[str, str]],
hf_token: gr.OAuthToken,
):
client = InferenceClient(token=hf_token.token, model="Qwen/Qwen3-8B", provider="featherless-ai")
messages = []
messages.extend(history)
messages.append({"role": "user", "content": message})
response = ""
for message in client.chat_completion(
messages,
stream=True,
):
choices = message.choices
token = ""
if len(choices) and choices[0].delta.content:
print("choices", choices)
token = choices[0].delta.content
print("token", token)
response += token
yield response
chatbot = gr.ChatInterface(
respond,
chatbot=gr.Chatbot(collapse_thinking=[("<think>", "</think>")])
)
with gr.Blocks() as demo:
with gr.Sidebar():
gr.LoginButton()
chatbot.render()
if __name__ == "__main__":
demo.launch()
|