.vscode/settings.json

@@ -7,9 +7,9 @@
         "**/.DS_Store": true,
         "**/Thumbs.db": true,
         "**/__pycache__": true,
+        "**/*.css.map": true,
         "**/.vscode": true,
         "**/*.map": true,
-        "**/*.css.map": true,
         "tests/__pycache__": true
     },
     "hide-files.files": [

api/config/config.yml

@@ -6,9 +6,10 @@ costs:
   other: 5
 
   chat-models:
-    gpt-4-32k: 200
+    gpt-4-32k-azure: 100
     gpt-4: 50
-    gpt-3: 10
+    gpt-4-azure: 10
+    gpt-3: 5
 
 ## Roles Explanation
 

api/helpers/tokens.py

@@ -38,10 +38,10 @@ async def count_for_messages(messages: list, model: str='gpt-3.5-turbo-0613') ->
         tokens_per_name = -1  # if there's a name, the role is omitted
 
     elif 'gpt-3.5-turbo' in model:
-        return await count_for_messages(messages, model='gpt-3.5-turbo-0613')
+        return count_for_messages(messages, model='gpt-3.5-turbo-0613')
 
     elif 'gpt-4' in model:
-        return await count_for_messages(messages, model='gpt-4-0613')
+        return count_for_messages(messages, model='gpt-4-0613')
 
     else:
         raise NotImplementedError(f"""count_for_messages() is not implemented for model {model}.

api/providers/azure.py

@@ -14,7 +14,7 @@ MODELS = [
 ]
 # MODELS = [f'{model}-azure' for model in MODELS]
 
-AZURE_API = '2023-08-01-preview'
+AZURE_API = '2023-07-01-preview'
 
 async def chat_completion(**payload):
     key = await utils.random_secret_for('azure-nva1')

api/responder.py

@@ -7,7 +7,6 @@ import aiohttp
 import asyncio
 import starlette
 
-from typing import Any, Coroutine, Set
 from rich import print
 from dotenv import load_dotenv
 
@@ -24,19 +23,6 @@ CRITICAL_API_ERRORS = ['invalid_api_key', 'account_deactivated']
 
 keymanager = providerkeys.manager
 
-background_tasks: Set[asyncio.Task[Any]] = set()
-
-
-def create_background_task(coro: Coroutine[Any, Any, Any]) -> None:
-    """asyncio.create_task, which prevents the task from being garbage collected.
-
-    https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task
-    """
-    task = asyncio.create_task(coro)
-    background_tasks.add(task)
-    task.add_done_callback(background_tasks.discard)
-
-
 async def respond(
     path: str='/v1/chat/completions',
     user: dict=None,
@@ -63,7 +49,7 @@ async def respond(
         'Content-Type': 'application/json'
     }
 
-    for i in range(5):
+    for i in range(1):
         try:
             if is_chat:
                 target_request = await load_balancing.balance_chat_request(payload)
@@ -110,7 +96,7 @@ async def respond(
                     cookies=target_request.get('cookies'),
                     ssl=False,
                     timeout=aiohttp.ClientTimeout(
-                        connect=0.75,
+                        connect=1.0,
                         total=float(os.getenv('TRANSFER_TIMEOUT', '500'))
                     )
                 ) as response:
@@ -161,33 +147,23 @@ async def respond(
                                 print('[!] too many requests')
                                 continue
 
-                        chunk_no = 0
-                        buffer = ''
-
-                        async for chunk  in response.content.iter_chunked(1024):
-                            chunk_no += 1
-
-                            chunk = chunk.decode('utf8')
+                        async for chunk in response.content.iter_any():
+                            chunk = chunk.decode('utf8').strip()
 
                             if 'azure' in provider_name:
-                                chunk = chunk.replace('data: ', '')
+                                chunk = chunk.strip().replace('data: ', '')
 
-                                if not chunk or chunk_no == 1:
+                                if not chunk or 'prompt_filter_results' in chunk:
                                     continue
 
-                            subchunks = chunk.split('\n\n')
-                            buffer += subchunks[0]
-
-                            yield buffer + '\n\n'
-                            buffer = subchunks[-1]
-
-                            for subchunk in subchunks[1:-1]:
-                                yield subchunk + '\n\n'
+                            yield chunk + '\n\n'
 
                     break
 
-            except aiohttp.client_exceptions.ServerTimeoutError:
-                continue
+            except Exception as exc:
+                print('[!] exception', exc)
+                # continue
+                raise exc
 
     else:
         yield await errors.yield_error(500, 'Sorry, our API seems to have issues connecting to our provider(s).', 'This most likely isn\'t your fault. Please try again later.')
@@ -196,7 +172,7 @@ async def respond(
     if (not is_stream) and server_json_response:
         yield json.dumps(server_json_response)
 
-    create_background_task(
+    asyncio.create_task(
         after_request.after_request(
             incoming_request=incoming_request,
             target_request=target_request,

checks/client.py

@@ -216,7 +216,7 @@ async def demo():
             raise ConnectionError('API Server is not running.')
 
         for func in [
-            test_chat_non_stream_gpt4,
+            # test_chat_non_stream_gpt4,
             test_chat_stream_gpt3
         ]:
             print(f'[*] {func.__name__}')