Refactor file operations

api/helpers/tokens.py

@@ -38,10 +38,10 @@ async def count_for_messages(messages: list, model: str='gpt-3.5-turbo-0613') ->
         tokens_per_name = -1  # if there's a name, the role is omitted
 
     elif 'gpt-3.5-turbo' in model:
-        return await count_for_messages(messages, model='gpt-3.5-turbo-0613')
+        return count_for_messages(messages, model='gpt-3.5-turbo-0613')
 
     elif 'gpt-4' in model:
-        return await count_for_messages(messages, model='gpt-4-0613')
+        return count_for_messages(messages, model='gpt-4-0613')
 
     else:
         raise NotImplementedError(f"""count_for_messages() is not implemented for model {model}.

api/providers/azure.py

@@ -14,7 +14,7 @@ MODELS = [
 ]
 # MODELS = [f'{model}-azure' for model in MODELS]
 
-AZURE_API = '2023-08-01-preview'
+AZURE_API = '2023-07-01-preview'
 
 async def chat_completion(**payload):
     key = await utils.random_secret_for('azure-nva1')

api/responder.py

@@ -7,7 +7,6 @@ import aiohttp
 import asyncio
 import starlette
 
-from typing import Any, Coroutine, Set
 from rich import print
 from dotenv import load_dotenv
 
@@ -24,19 +23,6 @@ CRITICAL_API_ERRORS = ['invalid_api_key', 'account_deactivated']
 
 keymanager = providerkeys.manager
 
-background_tasks: Set[asyncio.Task[Any]] = set()
-
-
-def create_background_task(coro: Coroutine[Any, Any, Any]) -> None:
-    """asyncio.create_task, which prevents the task from being garbage collected.
-
-    https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task
-    """
-    task = asyncio.create_task(coro)
-    background_tasks.add(task)
-    task.add_done_callback(background_tasks.discard)
-
-
 async def respond(
     path: str='/v1/chat/completions',
     user: dict=None,
@@ -63,7 +49,7 @@ async def respond(
         'Content-Type': 'application/json'
     }
 
-    for i in range(5):
+    for i in range(1):
         try:
             if is_chat:
                 target_request = await load_balancing.balance_chat_request(payload)
@@ -110,7 +96,7 @@ async def respond(
                     cookies=target_request.get('cookies'),
                     ssl=False,
                     timeout=aiohttp.ClientTimeout(
-                        connect=0.75,
+                        connect=1.0,
                         total=float(os.getenv('TRANSFER_TIMEOUT', '500'))
                     )
                 ) as response:
@@ -162,32 +148,24 @@ async def respond(
                                 continue
 
                         chunk_no = 0
-                        buffer = ''
+                        async for chunk in response.content.iter_any():
-
-                        async for chunk  in response.content.iter_chunked(1024):
                             chunk_no += 1
-
+                            chunk = chunk.decode('utf8').strip()
-                            chunk = chunk.decode('utf8')
 
                             if 'azure' in provider_name:
-                                chunk = chunk.replace('data: ', '')
+                                chunk = chunk.strip().replace('data: ', '')
 
                                 if not chunk or chunk_no == 1:
                                     continue
 
-                            subchunks = chunk.split('\n\n')
+                            yield chunk + '\n\n'
-                            buffer += subchunks[0]
-
-                            yield buffer + '\n\n'
-                            buffer = subchunks[-1]
-
-                            for subchunk in subchunks[1:-1]:
-                                yield subchunk + '\n\n'
 
                     break
 
-            except aiohttp.client_exceptions.ServerTimeoutError:
+            except Exception as exc:
-                continue
+                print('[!] exception', exc)
+                # continue
+                raise exc
 
     else:
         yield await errors.yield_error(500, 'Sorry, our API seems to have issues connecting to our provider(s).', 'This most likely isn\'t your fault. Please try again later.')
@@ -196,7 +174,7 @@ async def respond(
     if (not is_stream) and server_json_response:
         yield json.dumps(server_json_response)
 
-    create_background_task(
+    asyncio.create_task(
         after_request.after_request(
             incoming_request=incoming_request,
             target_request=target_request,