WIP

api/main.py

@@ -7,7 +7,6 @@ from fastapi.middleware.cors import CORSMiddleware
 
 from dotenv import load_dotenv
 
-import security
 import transfer
 
 load_dotenv()
@@ -25,7 +24,6 @@ app.add_middleware(
 @app.on_event('startup')
 async def startup_event():
     """Read up the API server."""
-    # await security.ip_protection_check()
 
 @app.get('/')
 async def root():
@@ -44,4 +42,4 @@ async def api_root():
         'status': 'ok',
     }
 
-app.add_route('/{path:path}', transfer.transfer_streaming_response, ['GET', 'POST', 'PUT', 'DELETE', 'PATCH'])
+app.add_route('/{path:path}', transfer.handle_api_request, ['GET', 'POST', 'PUT', 'DELETE', 'PATCH'])

api/netclient.py

@@ -0,0 +1,24 @@
+import os
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+async def receive_target_stream():
+    async with aiohttp.ClientSession(
+        timeout=aiohttp.ClientTimeout(total=int(os.getenv('TRANSFER_TIMEOUT', '120'))),
+        raise_for_status=False
+    ) as session:     
+        async with session.request(
+            method=incoming_request.method,
+            url=target_url,
+            json=incoming_json_payload,
+            headers={
+                'Content-Type': 'application/json',
+                'Authorization': f'Bearer {os.getenv("CLOSEDAI_KEY")}'
+            }
+        ) as response:
+            async for chunk in response.content.iter_any():
+                chunk = f'{chunk.decode("utf8")}\n\n'
+
+                yield chunk

api/proxies.py

@@ -3,6 +3,8 @@
 import os
 import socket
 import asyncio
+import aiohttp
+import aiohttp_socks
 
 from dotenv import load_dotenv
 
@@ -25,10 +27,51 @@ class Proxy:
         self.username = username
         self.password = password
 
-active_proxy = Proxy(
+    async def initialize_connector(self, connector):
+        async with aiohttp.ClientSession(
+            connector=connector,
+            timeout=aiohttp.ClientTimeout(total=10),
+            raise_for_status=True
+        ) as session:
+            async with session.request(
+                method='get',
+                url='https://checkip.amazonaws.com',
+                headers={'Content-Type': 'application/json'}
+            ) as response:
+                detected_ip = await response.text()
+                print(f'Detected IP: {detected_ip}')
+                return detected_ip.strip()
+
+    async def get_connector(self):
+        proxy_types = {
+            'http': aiohttp_socks.ProxyType.HTTP,
+            'https': aiohttp_socks.ProxyType.HTTP,
+            'socks4': aiohttp_socks.ProxyType.SOCKS4,
+            'socks5': aiohttp_socks.ProxyType.SOCKS5
+        }
+
+        connector = aiohttp_socks.ProxyConnector(
+            proxy_type=proxy_types[self.proxy_type],
+            host=self.ip_address,
+            port=self.port,
+            rdns=False,
+            username=self.username,
+            password=self.password
+        )
+
+        await self.initialize_connector(connector)
+
+        # Logging to check the connector state
+        print("Connector: Is closed?", connector.closed)
+        print("Connector: Is connected?", connector._connected)
+
+        return connector
+
+
+default_proxy = Proxy(
     proxy_type=os.getenv('PROXY_TYPE', 'http'),
     host=os.getenv('PROXY_HOST', '127.0.0.1'),
-    port=int(os.getenv('PROXY_PORT', 8080)),
+    port=int(os.getenv('PROXY_PORT', '8080')),
     username=os.getenv('PROXY_USER'),
     password=os.getenv('PROXY_PASS')
 )

api/security.py

@@ -1,45 +0,0 @@
-"""Security checks for the API. Checks if the IP is masked etc."""
-
-import os
-import asyncio
-
-from rich import print
-from tests import check_proxy
-from dotenv import load_dotenv
-
-load_dotenv()
-is_proxy_enabled = bool(os.getenv('PROXY_HOST', None))
-
-class InsecureIPError(Exception):
-    """Raised when the IP address of the server is not secure."""
-
-async def ip_protection_check():
-    """Makes sure that the actual server IP address is not exposed to the public."""
-
-    if not is_proxy_enabled:
-        print('[yellow]WARN:     The proxy is not enabled. \
-Skipping IP protection check.[/yellow]')
-        return True
-
-    actual_ips = os.getenv('ACTUAL_IPS', '').split()
-
-    if actual_ips:
-        # run the async function check_proxy() and get its result
-        response_ip = await check_proxy()
-
-        for actual_ip in actual_ips:
-            if actual_ip in response_ip:
-                raise InsecureIPError(f'IP pattern "{actual_ip}" is in the values of ACTUAL_IPS of the\
-.env file. Enable a VPN or proxy to continue.')
-
-        if is_proxy_enabled:
-            print(f'[green]GOOD:     The IP "{response_ip}" was detected, which seems to be a proxy. Great![/green]')
-            return True
-
-    else:
-        print('[yellow]WARN:     ACTUAL_IPS is not set in the .env file or empty.\
-This means that the real IP of the server could be exposed. If you\'re using something\
-like Cloudflare or Repl.it, you can ignore this warning.[/yellow]')
-
-if __name__ == '__main__':
-    ip_protection_check()

api/tests.py

@@ -1,62 +0,0 @@
-import os
-import httpx
-import proxies
-import asyncio
-import aiohttp
-import aiohttp_socks
-
-from rich import print
-from dotenv import load_dotenv
-
-load_dotenv()
-
-async def check_proxy():
-    """Checks if the proxy is working."""
-
-    proxy = proxies.active_proxy
-
-    connector = aiohttp_socks.ProxyConnector(
-        proxy_type=aiohttp_socks.ProxyType.SOCKS5,
-        host=proxy.ip_address,
-        port=proxy.port,
-        rdns=False,
-        username=proxy.username,
-        password=proxy.password
-    )
-
-    async with aiohttp.ClientSession(connector=connector) as session:
-        async with session.get('https://echo.hoppscotch.io/') as response:
-            json_data = await response.json()
-            return json_data['headers']['x-forwarded-for']
-
-async def check_api():
-    """Checks if the API is working."""
-
-    model = 'gpt-3.5-turbo'
-    messages = [
-        {
-            'role': 'user',
-            'content': '2+2='
-        },
-    ]
-
-    headers = {
-        'Content-Type': 'application/json',
-        'Authorization': 'Bearer '  + os.getenv('CLOSEDAI_KEY')
-    }
-
-    json_data = {
-        'model': model,
-        'messages': messages,
-        'stream': True
-    }
-
-    async with httpx.AsyncClient(timeout=20) as client:
-        async with client.stream('POST', 'https://api.openai.com/v1/chat/completions', headers=headers, json=json_data) as response:
-            response.raise_for_status()
-            async for chunk in response.aiter_text():
-                print(chunk.strip())
-
-if __name__ == '__main__':
-    # asyncio.run(check_api())
-    asyncio.run(check_proxy())

api/transfer.py

@@ -2,17 +2,16 @@
 
 import os
 import json
-import logging
 import aiohttp
-import aiohttp_socks
+import logging
+import starlette
+import netclient
 
 import proxies
 
 from dotenv import load_dotenv
 
-import starlette
+from starlette.background import StreamingResponse
-from starlette.responses import StreamingResponse
-from starlette.background import BackgroundTask
 
 load_dotenv()
 
@@ -32,61 +31,30 @@ EXCLUDED_HEADERS = [
     'connection'
 ]
 
-proxy = proxies.active_proxy
+async def handle_api_request(incoming_request, target_endpoint: str=''):
-proxy_connector = aiohttp_socks.ProxyConnector(
-    proxy_type=aiohttp_socks.ProxyType.SOCKS5,
-    host=proxy.ip_address,
-    port=proxy.port,
-    rdns=False,
-    username=proxy.username,
-    password=proxy.password
-)
-
-async def transfer_streaming_response(incoming_request, target_endpoint: str='https://api.openai.com/v1'):
     """Transfer a streaming response from the incoming request to the target endpoint"""
+    if not target_endpoint:
+        target_endpoint = os.getenv('CLOSEDAI_ENDPOINT')
 
-    if incoming_request.headers.get('Authorization') != f'Bearer {os.getenv("DEMO_AUTH")}':
-        return starlette.responses.Response(
-            status_code=403,
-            content='Invalid API key'
-        )
-
-    try:
-        incoming_json_payload = await incoming_request.json()
-    except json.decoder.JSONDecodeError:
-        incoming_json_payload = {}
-
-    async def receive_target_stream():
-        connector = aiohttp_socks.ProxyConnector(
-            proxy_type=aiohttp_socks.ProxyType.SOCKS5,
-            host=proxy.ip_address,
-            port=proxy.port,
-            rdns=False,
-            username=proxy.username,
-            password=proxy.password
-        )
-        async with aiohttp.ClientSession(
-            connector=connector,
-            timeout=aiohttp.ClientTimeout(total=int(os.getenv('TRANSFER_TIMEOUT', '120'))),
-            raise_for_status=True
-        ) as session:
     target_url = f'{target_endpoint}{incoming_request.url.path}'.replace('/v1/v1', '/v1')
     logging.info('TRANSFER %s -> %s', incoming_request.url.path, target_url)
 
-            async with session.request(
+    if target_url.endswith('/v1'):
-                method=incoming_request.method,
+        return starlette.responses.Response(status_code=200, content='200. /v1 is the API endpoint root path.')
-                url=target_url,
+
-                json=incoming_json_payload,
+    if incoming_request.headers.get('Authorization') != f'Bearer {os.getenv("DEMO_AUTH")}':
-                headers={
+        return starlette.responses.Response(status_code=403, content='Invalid API key')
-                    'Content-Type': 'application/json',
+
-                    'Authorization': f'Bearer {os.getenv("CLOSEDAI_KEY")}'
+    try:
-                }
+        payload = await incoming_request.json()
-            ) as response:
+    except json.decoder.JSONDecodeError:
-                async for chunk in response.content.iter_any():
+        payload = {}
-                    chunk = f'{chunk.decode("utf8")}\n\n'
+
-                    logging.debug(chunk)
+    target_provider = 'moe'
-                    yield chunk
+
+    if 'temperature' in payload or 'functions' in payload:
+        target_provider = 'closed'
 
     return StreamingResponse(
-        content=receive_target_stream()
+        content=netclient.receive_target_stream()
     )

proxy_exception.txt

@@ -0,0 +1,64 @@
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.10/site-packages/uvicorn/protocols/http/h11_impl.py", line 429, in run_asgi
+    result = await app(  # type: ignore[func-returns-value]
+  File "/usr/local/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
+    return await self.app(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/fastapi/applications.py", line 284, in __call__
+    await super().__call__(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/applications.py", line 122, in __call__
+    await self.middleware_stack(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/errors.py", line 184, in __call__
+    raise exc
+  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__
+    await self.app(scope, receive, _send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/cors.py", line 83, in __call__
+    await self.app(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
+    raise exc
+  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
+    await self.app(scope, receive, sender)
+  File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 20, in __call__
+    raise e
+  File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 17, in __call__
+    await self.app(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__
+    await route.handle(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle
+    await self.app(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 69, in app
+    await response(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/responses.py", line 270, in __call__
+    async with anyio.create_task_group() as task_group:
+  File "/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 662, in __aexit__
+    raise exceptions[0]
+  File "/usr/local/lib/python3.10/site-packages/starlette/responses.py", line 273, in wrap
+    await func()
+  File "/usr/local/lib/python3.10/site-packages/starlette/responses.py", line 262, in stream_response
+    async for chunk in self.body_iterator:
+  File "/home/nova-api/api/transfer.py", line 62, in receive_target_stream
+    async with session.request(
+  File "/usr/local/lib/python3.10/site-packages/aiohttp/client.py", line 1138, in __aenter__
+    self._resp = await self._coro
+  File "/usr/local/lib/python3.10/site-packages/aiohttp/client.py", line 535, in _request
+    conn = await self._connector.connect(
+  File "/usr/local/lib/python3.10/site-packages/aiohttp/connector.py", line 542, in connect
+    proto = await self._create_connection(req, traces, timeout)
+  File "/usr/local/lib/python3.10/site-packages/aiohttp/connector.py", line 907, in _create_connection
+    _, proto = await self._create_direct_connection(req, traces, timeout)
+  File "/usr/local/lib/python3.10/site-packages/aiohttp/connector.py", line 1175, in _create_direct_connection
+    transp, proto = await self._wrap_create_connection(
+  File "/usr/local/lib/python3.10/site-packages/aiohttp_socks/connector.py", line 72, in _wrap_create_connection
+    stream = await proxy.connect(
+  File "/usr/local/lib/python3.10/site-packages/python_socks/async_/asyncio/v2/_proxy.py", line 59, in connect
+    return await self._connect(
+  File "/usr/local/lib/python3.10/site-packages/python_socks/async_/asyncio/v2/_proxy.py", line 100, in _connect
+    await self._negotiate(
+  File "/usr/local/lib/python3.10/site-packages/python_socks/async_/asyncio/v2/_proxy.py", line 164, in _negotiate
+    await proto.negotiate()
+  File "/usr/local/lib/python3.10/site-packages/python_socks/_proto/socks5_async.py", line 42, in negotiate
+    await self._socks_auth()
+  File "/usr/local/lib/python3.10/site-packages/python_socks/_proto/socks5_async.py", line 55, in _socks_auth
+    res.validate()
+  File "/usr/local/lib/python3.10/site-packages/python_socks/_proto/socks5.py", line 134, in validate
+    raise ProxyError('Username and password '  # pragma: no cover
+python_socks._errors.ProxyError: Username and password authentication failure

session_exception.txt

@@ -0,0 +1,44 @@
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.10/site-packages/uvicorn/protocols/http/h11_impl.py", line 429, in run_asgi
+    result = await app(  # type: ignore[func-returns-value]
+  File "/usr/local/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
+    return await self.app(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/fastapi/applications.py", line 284, in __call__
+    await super().__call__(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/applications.py", line 122, in __call__
+    await self.middleware_stack(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/errors.py", line 184, in __call__
+    raise exc
+  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__
+    await self.app(scope, receive, _send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/cors.py", line 83, in __call__
+    await self.app(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
+    raise exc
+  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
+    await self.app(scope, receive, sender)
+  File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 20, in __call__
+    raise e
+  File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 17, in __call__
+    await self.app(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__
+    await route.handle(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle
+    await self.app(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 69, in app
+    await response(scope, receive, send)
+  File "/usr/local/lib/python3.10/site-packages/starlette/responses.py", line 270, in __call__
+    async with anyio.create_task_group() as task_group:
+  File "/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 662, in __aexit__
+    raise exceptions[0]
+  File "/usr/local/lib/python3.10/site-packages/starlette/responses.py", line 273, in wrap
+    await func()
+  File "/usr/local/lib/python3.10/site-packages/starlette/responses.py", line 262, in stream_response
+    async for chunk in self.body_iterator:
+  File "/home/nova-api/api/transfer.py", line 60, in receive_target_stream
+    async with session.request(
+  File "/usr/local/lib/python3.10/site-packages/aiohttp/client.py", line 1138, in __aenter__
+    self._resp = await self._coro
+  File "/usr/local/lib/python3.10/site-packages/aiohttp/client.py", line 399, in _request
+    raise RuntimeError("Session is closed")
+RuntimeError: Session is closed