Update app.py

app.py

@@ -24,12 +24,12 @@ llm_model = None
 
 hf_hub_download(
     repo_id="bartowski/Qwen2.5-Coder-1.5B-Instruct-GGUF",
-    filename="Qwen2.5-Coder-1.5B-Instruct-Q6_K.gguf",
+    filename="Qwen2.5-Coder-1.5B-Instruct-Q4_K_M.gguf",
     local_dir="./models",
 )
 hf_hub_download(
     repo_id="bartowski/Qwen2.5-Coder-0.5B-Instruct-GGUF",
-    filename="Qwen2.5-Coder-0.5B-Instruct-Q6_K.gguf",
+    filename="Qwen2.5-Coder-0.5B-Instruct-Q4_K_M.gguf",
     local_dir="./models",
 )
 
@@ -77,8 +77,10 @@ def respond(
                 model_path=f"models/{model}",
                 flash_attn=False,
                 n_gpu_layers=0,
-                n_batch=16,
+                n_batch=8,
                 n_ctx=2048,
+                n_threads=2,
+                n_threads_batch=2,
             )
             llm_model = model
         provider = LlamaCppPythonProvider(llm)
@@ -143,10 +145,10 @@ demo = gr.ChatInterface(
     additional_inputs=[
         gr.Dropdown(
             choices=[
-                "Qwen2.5-Coder-1.5B-Instruct-Q6_K.gguf",
-                "Qwen2.5-Coder-0.5B-Instruct-Q6_K.gguf",
+                "Qwen2.5-Coder-1.5B-Instruct-Q4_K_M.gguf",
+                "Qwen2.5-Coder-0.5B-Instruct-Q4_K_M.gguf",
             ],
-            value="Qwen2.5-Coder-0.5B-Instruct-Q6_K.gguf",
+            value="Qwen2.5-Coder-0.5B-Instruct-Q4_K_M.gguf",
             label="Model",
             info="Select the AI model to use for chat",
         ),