Fix dangling asyncio tasks

.vscode/settings.json

@@ -7,9 +7,9 @@
         "**/.DS_Store": true,
         "**/Thumbs.db": true,
         "**/__pycache__": true,
-        "**/*.css.map": true,
         "**/.vscode": true,
         "**/*.map": true,
+        "**/*.css.map": true,
         "tests/__pycache__": true
     },
     "hide-files.files": [

api/config/config.yml

@@ -6,10 +6,9 @@ costs:
   other: 5
 
   chat-models:
-    gpt-4-32k-azure: 100
+    gpt-4-32k: 200
     gpt-4: 50
-    gpt-4-azure: 10
+    gpt-3: 10
-    gpt-3: 5
 
 ## Roles Explanation
 

api/responder.py

@@ -7,6 +7,7 @@ import aiohttp
 import asyncio
 import starlette
 
+from typing import Any, Coroutine, Set
 from rich import print
 from dotenv import load_dotenv
 
@@ -23,6 +24,19 @@ CRITICAL_API_ERRORS = ['invalid_api_key', 'account_deactivated']
 
 keymanager = providerkeys.manager
 
+background_tasks: Set[asyncio.Task[Any]] = set()
+
+
+def create_background_task(coro: Coroutine[Any, Any, Any]) -> None:
+    """asyncio.create_task, which prevents the task from being garbage collected.
+
+    https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task
+    """
+    task = asyncio.create_task(coro)
+    background_tasks.add(task)
+    task.add_done_callback(background_tasks.discard)
+
+
 async def respond(
     path: str='/v1/chat/completions',
     user: dict=None,
@@ -147,13 +161,15 @@ async def respond(
                                 print('[!] too many requests')
                                 continue
 
+                        chunk_no = 0
                         async for chunk in response.content.iter_any():
+                            chunk_no += 1
                             chunk = chunk.decode('utf8').strip()
 
                             if 'azure' in provider_name:
                                 chunk = chunk.strip().replace('data: ', '')
 
-                                if not chunk or 'prompt_filter_results' in chunk:
+                                if not chunk or chunk_no == 1:
                                     continue
 
                             yield chunk + '\n\n'
@@ -172,7 +188,7 @@ async def respond(
     if (not is_stream) and server_json_response:
         yield json.dumps(server_json_response)
 
-    asyncio.create_task(
+    create_background_task(
         after_request.after_request(
             incoming_request=incoming_request,
             target_request=target_request,

checks/client.py

@@ -216,7 +216,7 @@ async def demo():
             raise ConnectionError('API Server is not running.')
 
         for func in [
-            # test_chat_non_stream_gpt4,
+            test_chat_non_stream_gpt4,
             test_chat_stream_gpt3
         ]:
             print(f'[*] {func.__name__}')