Update main.py

main.py

@@ -9,7 +9,8 @@ from fastapi import HTTPException, Depends, Request
 from fastapi.responses import HTMLResponse, StreamingResponse
 from fastapi.middleware.cors import CORSMiddleware
 from sse_starlette.sse import EventSourceResponse
-from anyio import create_memory_object_stream, run_in_threadpool
+from anyio import create_memory_object_stream
+from anyio.to_thread import run_sync
 from transformers import AutoModelForCausalLM
 from pydantic import BaseModel
 
@@ -101,7 +102,7 @@ async def chatV2(request: Request, body: ChatCompletionRequest):
     async def event_publisher(inner_send_chan):
         async with inner_send_chan:
             try:
-                iterator: Generator = await run_in_threadpool(llm.generate, tokens)
+                iterator: Generator = await run_sync(llm.generate, tokens)
                 for chat_chunk in iterator:
                     response = {
                         'choices': [
@@ -118,10 +119,9 @@ async def chatV2(request: Request, body: ChatCompletionRequest):
                 await inner_send_chan.send("event: done\ndata: {}\n\n")
             except Exception as e:
                 print(f"Exception in event publisher: {str(e)}")
-
+    
     return StreamingResponse(recv_chan, media_type="text/event-stream", data_sender_callable=partial(event_publisher, send_chan))
 
-
 @app.post("/v0/chat/completions")
 async def chat(request: ChatCompletionRequestV0, response_mode=None):
     tokens = llm.tokenize(request.prompt)