From a7bd57c5a80580c2a5aebf7b1d6c8a229a00dbc5 Mon Sep 17 00:00:00 2001 From: nsde Date: Sat, 12 Aug 2023 17:49:31 +0200 Subject: [PATCH] Added more documentation --- .gitignore | 3 --- api/core.py | 11 --------- api/helpers/chat.py | 4 +++ api/helpers/errors.py | 6 ++++- api/helpers/exceptions.py | 2 -- api/helpers/network.py | 4 +++ api/helpers/tokens.py | 6 ++--- api/load_balancing.py | 7 ++++-- api/moderation.py | 13 +++++++++- api/provider_auth.py | 10 +++++++- api/proxies.py | 52 ++++++++++++++++++++++++++------------- api/streaming.py | 41 +++++++++++++++++++++--------- api/transfer.py | 2 +- setup.md | 30 ++++++++++++++++++++-- 14 files changed, 136 insertions(+), 55 deletions(-) delete mode 100644 api/helpers/exceptions.py diff --git a/.gitignore b/.gitignore index 9abd447..08e41d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,5 @@ last_update.txt -!api/providers/__main__.py -!api/providers/helpers/utils.py - *.log.json /logs /log diff --git a/api/core.py b/api/core.py index f266147..fe16cd9 100644 --- a/api/core.py +++ b/api/core.py @@ -63,14 +63,3 @@ async def create_user(incoming_request: fastapi.Request): await new_user_webhook(user) return user - -if __name__ == '__main__': - # new_user_webhook({ - # '_id': 'JUST_A_TEST_IGNORE_ME', - # 'auth': { - # 'discord': 123, - # 'github': 'abc' - # } - # }) - - pass diff --git a/api/helpers/chat.py b/api/helpers/chat.py index a1002fc..9cd35aa 100644 --- a/api/helpers/chat.py +++ b/api/helpers/chat.py @@ -12,12 +12,16 @@ class CompletionStop: """End of a chat""" async def create_chat_id() -> str: + """Generates a random chat ID""" + chars = string.ascii_letters + string.digits chat_id = ''.join(random.choices(chars, k=32)) return f'chatcmpl-{chat_id}' async def create_chat_chunk(chat_id: str, model: str, content=None) -> dict: + """Creates a new chat chunk""" + content = content or {} delta = {} diff --git a/api/helpers/errors.py b/api/helpers/errors.py index de4f3e5..a4bee00 100644 --- a/api/helpers/errors.py +++ b/api/helpers/errors.py @@ -1,7 +1,9 @@ import json import starlette -async def error(code: int, message: str, tip: str) -> starlette.responses.Response: +async def error(code: int, message: str, tip: str) -> starlette.responses.Response: + """Returns a starlette response JSON with the given error code, message and tip.""" + info = {'error': { 'code': code, 'message': message, @@ -13,6 +15,8 @@ async def error(code: int, message: str, tip: str) -> starlette.responses.Respon return starlette.responses.Response(status_code=code, content=json.dumps(info)) async def yield_error(code: int, message: str, tip: str) -> str: + """Returns a dumped JSON response with the given error code, message and tip.""" + return json.dumps({ 'code': code, 'message': message, diff --git a/api/helpers/exceptions.py b/api/helpers/exceptions.py deleted file mode 100644 index 1fcff8b..0000000 --- a/api/helpers/exceptions.py +++ /dev/null @@ -1,2 +0,0 @@ -class Retry(Exception): - """The server should retry the request.""" diff --git a/api/helpers/network.py b/api/helpers/network.py index 76a3bad..ce67410 100644 --- a/api/helpers/network.py +++ b/api/helpers/network.py @@ -2,6 +2,8 @@ import base64 import asyncio async def get_ip(request) -> str: + """Get the IP address of the incoming request.""" + xff = None if request.headers.get('x-forwarded-for'): xff, *_ = request.headers['x-forwarded-for'].split(', ') @@ -17,6 +19,8 @@ async def get_ip(request) -> str: return detected_ip async def add_proxy_auth_to_headers(username: str, password: str, headers: dict) -> dict: + """Add proxy authentication to the headers. This is useful if the proxy authentication doesn't work as it should.""" + proxy_auth = base64.b64encode(f'{username}:{password}'.encode()).decode() headers['Proxy-Authorization'] = f'Basic {proxy_auth}' return headers diff --git a/api/helpers/tokens.py b/api/helpers/tokens.py index e1aede6..10b3dd0 100644 --- a/api/helpers/tokens.py +++ b/api/helpers/tokens.py @@ -25,13 +25,13 @@ async def count_for_messages(messages: list, model: str='gpt-3.5-turbo-0613') -> tokens_per_name = -1 # if there's a name, the role is omitted elif 'gpt-3.5-turbo' in model: - return num_tokens_from_messages(messages, model='gpt-3.5-turbo-0613') + return count_for_messages(messages, model='gpt-3.5-turbo-0613') elif 'gpt-4' in model: - return num_tokens_from_messages(messages, model='gpt-4-0613') + return count_for_messages(messages, model='gpt-4-0613') else: - raise NotImplementedError(f"""num_tokens_from_messages() is not implemented for model {model}. + raise NotImplementedError(f"""count_for_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""") diff --git a/api/load_balancing.py b/api/load_balancing.py index 3fb6f54..cfcfe72 100644 --- a/api/load_balancing.py +++ b/api/load_balancing.py @@ -10,7 +10,8 @@ async def _get_module_name(module) -> str: return name async def balance_chat_request(payload: dict) -> dict: - """Load balance the chat completion request between chat providers.""" + """Load balance the chat completion request between chat providers. +""" providers_available = [] @@ -35,7 +36,9 @@ async def balance_chat_request(payload: dict) -> dict: return target async def balance_organic_request(request: dict) -> dict: - """Load balnace to non-chat completion request between other "organic" providers which respond in the desired format already.""" + """Load balnace to non-chat completion request between other "organic" providers which respond in the desired format already. +Organic providers are used for non-chat completions, such as moderation and other paths. +""" providers_available = [] diff --git a/api/moderation.py b/api/moderation.py index d0c8144..bdc9ff3 100644 --- a/api/moderation.py +++ b/api/moderation.py @@ -1,3 +1,5 @@ +"""This module contains functions for checking if a message violates the moderation policy.""" + import asyncio import aiohttp @@ -5,7 +7,16 @@ import proxies import provider_auth import load_balancing -async def is_policy_violated(inp) -> bool: +from typing import Union + +async def is_policy_violated(inp: Union[str, list]) -> bool: + """Check if a message violates the moderation policy. +You can either pass a list of messages consisting of dicts with "role" and "content", as used in the API parameter, +or just a simple string. + +Returns True if the message violates the policy, False otherwise. +""" + text = inp if isinstance(inp, list): diff --git a/api/provider_auth.py b/api/provider_auth.py index 9393109..bc499d7 100644 --- a/api/provider_auth.py +++ b/api/provider_auth.py @@ -1,6 +1,14 @@ +"""This module contains functions for authenticating with providers.""" + import asyncio -async def invalidate_key(provider_and_key): +async def invalidate_key(provider_and_key: str) -> none: + """Invalidates a key stored in the secret/ folder by storing it in the associated .invalid.txt file. +The schmea in which should be passed is: +, e.g. +closed4>sk-... +""" + if not provider_and_key: return diff --git a/api/proxies.py b/api/proxies.py index 639607a..930b5f9 100644 --- a/api/proxies.py +++ b/api/proxies.py @@ -12,8 +12,14 @@ from dotenv import load_dotenv load_dotenv() +USE_PROXY_LIST = os.getenv('USE_PROXY_LIST', 'False').lower() == 'true' + class Proxy: - """Represents a proxy. The type can be either http, https, socks4 or socks5.""" + """Represents a proxy. The type can be either http, https, socks4 or socks5. +You can also pass a url, which will be parsed into the other attributes. +URL format: + [type]://[username:password@]host:port +""" def __init__(self, url: str=None, @@ -36,7 +42,7 @@ class Proxy: self.proxy_type = proxy_type self.host_or_ip = host_or_ip - self.ip_address = socket.gethostbyname(self.host_or_ip) #if host_or_ip.replace('.', '').isdigit() else host_or_ip + self.ip_address = socket.gethostbyname(self.host_or_ip) # get ip address from host self.host = self.host_or_ip self.port = port self.username = username @@ -54,6 +60,8 @@ class Proxy: @property def connector(self): + """Returns an aiohttp_socks.ProxyConnector object. Which can be used in aiohttp.ClientSession.""" + proxy_types = { 'http': aiohttp_socks.ProxyType.HTTP, 'https': aiohttp_socks.ProxyType.HTTP, @@ -65,11 +73,13 @@ class Proxy: proxy_type=proxy_types[self.proxy_type], host=self.ip_address, port=self.port, - rdns=False, + rdns=False, # remote DNS username=self.username, password=self.password ) +# load proxies from files + proxies_in_files = [] try: @@ -84,27 +94,19 @@ try: except FileNotFoundError: pass -class ProxyChain: +# Proxy lists support + +class ProxyLists: def __init__(self): random_proxy = random.choice(proxies_in_files) self.get_random = Proxy(url=random_proxy) self.connector = aiohttp_socks.ChainProxyConnector.from_urls(proxies_in_files) -try: - default_chain = ProxyChain() - random_proxy = ProxyChain().get_random -except IndexError: - pass - -default_proxy = Proxy( - proxy_type=os.getenv('PROXY_TYPE', 'http'), - host_or_ip=os.getenv('PROXY_HOST', '127.0.0.1'), - port=int(os.getenv('PROXY_PORT', '8080')), - username=os.getenv('PROXY_USER'), - password=os.getenv('PROXY_PASS') -) +# ================================================================================================================================ # +# Proxy tests +# Can be useful if you want to troubleshoot your proxies def test_httpx_workaround(): import httpx @@ -153,6 +155,22 @@ async def text_httpx_socks(): res = await client.get('https://checkip.amazonaws.com') return res.text +# ================================================================================================================================ # + +def get_proxy() -> Proxy: + """Returns a Proxy object. The proxy is either from the proxy list or from the environment variables. +""" + if USE_PROXY_LIST: + return ProxyLists().get_random + + return Proxy( + proxy_type=os.getenv('PROXY_TYPE', 'http'), + host_or_ip=os.getenv('PROXY_HOST', '127.0.0.1'), + port=int(os.getenv('PROXY_PORT', '8080')), + username=os.getenv('PROXY_USER'), + password=os.getenv('PROXY_PASS') + ) + if __name__ == '__main__': # print(test_httpx()) # print(test_requests()) diff --git a/api/streaming.py b/api/streaming.py index 09dc000..295b06b 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -1,3 +1,5 @@ +"""This module contains the streaming logic for the API.""" + import os import json import dhooks @@ -33,7 +35,6 @@ async def stream( user: dict=None, payload: dict=None, credits_cost: int=0, - demo_mode: bool=False, input_tokens: int=0, incoming_request: starlette.requests.Request=None, ): @@ -44,6 +45,7 @@ async def stream( is_chat = True model = payload['model'] + # Chat completions always have the same beginning if is_chat and is_stream: chat_id = await chat.create_chat_id() @@ -66,15 +68,22 @@ async def stream( 'error': 'No JSON response could be received' } + # Try to get a response from the API for _ in range(5): headers = { 'Content-Type': 'application/json' } + # Load balancing + # If the request is a chat completion, then we need to load balance between chat providers + # If the request is an organic request, then we need to load balance between organic providers + try: if is_chat: target_request = await load_balancing.balance_chat_request(payload) else: + # "organic" means that it's not using a reverse engineered front-end, but rather ClosedAI's API directly + # churchless.tech is an example of an organic provider, because it redirects the request to ClosedAI. target_request = await load_balancing.balance_organic_request({ 'method': incoming_request.method, 'path': path, @@ -83,9 +92,10 @@ async def stream( 'cookies': incoming_request.cookies }) except ValueError as exc: + # Error load balancing? Send a webhook to the admins webhook = dhooks.Webhook(os.getenv('DISCORD_WEBHOOK__API_ISSUE')) - webhook.send(content=f'API Issue: **`{exc}`**\nhttps://i.imgflip.com/7uv122.jpg') + error = await errors.yield_error( 500, 'Sorry, the API has no working keys anymore.', @@ -100,7 +110,9 @@ async def stream( if target_request['method'] == 'GET' and not payload: target_request['payload'] = None - async with aiohttp.ClientSession(connector=proxies.default_proxy.connector) as session: + # We haven't done any requests as of right now, everything until now was just preparation + # Here, we process the request + async with aiohttp.ClientSession(connector=proxies.get_proxy().connector) as session: try: async with session.request( method=target_request.get('method', 'POST'), @@ -116,9 +128,11 @@ async def stream( timeout=aiohttp.ClientTimeout(total=float(os.getenv('TRANSFER_TIMEOUT', '120'))), ) as response: + # if the answer is JSON if response.content_type == 'application/json': data = await response.json() + # Invalidate the key if it's not working if data.get('code') == 'invalid_api_key': await provider_auth.invalidate_key(target_request.get('provider_auth')) continue @@ -126,37 +140,40 @@ async def stream( if response.ok: json_response = data + # if the answer is a stream if is_stream: try: response.raise_for_status() except Exception as exc: + # Rate limit? Balance again if 'Too Many Requests' in str(exc): continue try: + # process the response chunks async for chunk in response.content.iter_any(): send = False chunk = f'{chunk.decode("utf8")}\n\n' - chunk = chunk.replace(os.getenv('MAGIC_WORD', 'novaOSScheckKeyword'), payload['model']) - chunk = chunk.replace(os.getenv('MAGIC_USER_WORD', 'novaOSSuserKeyword'), str(user['_id'])) if is_chat and '{' in chunk: + # parse the JSON data = json.loads(chunk.split('data: ')[1]) chunk = chunk.replace(data['id'], chat_id) send = True + # create a custom chunk if we're using specific providers if target_request['module'] == 'twa' and data.get('text'): chunk = await chat.create_chat_chunk( chat_id=chat_id, model=model, content=['text'] ) - if not data['choices'][0]['delta']: + + # don't send empty/unnecessary messages + if (not data['choices'][0]['delta']) or data['choices'][0]['delta'] == {'role': 'assistant'}: send = False - if data['choices'][0]['delta'] == {'role': 'assistant'}: - send = False - + # send the chunk if send and chunk.strip(): final_chunk = chunk.strip().replace('data: [DONE]', '') + '\n\n' yield final_chunk @@ -174,9 +191,10 @@ async def stream( break except ProxyError as exc: - print('proxy error') + print('[!] Proxy error:', exc) continue + # Chat completions always have the same ending if is_chat and is_stream: chunk = await chat.create_chat_chunk( chat_id=chat_id, @@ -186,10 +204,11 @@ async def stream( yield chunk yield 'data: [DONE]\n\n' + # If the response is JSON, then we need to yield it like this if not is_stream and json_response: yield json.dumps(json_response) - # DONE ========================================================= + # DONE WITH REQUEST, NOW LOGGING ETC. if user and incoming_request: await logs.log_api_request( diff --git a/api/transfer.py b/api/transfer.py index a4f83f6..23011f9 100644 --- a/api/transfer.py +++ b/api/transfer.py @@ -1,4 +1,4 @@ -"""Module for transferring requests to ClosedAI API""" +"""Does quite a few checks and prepares the incoming request for the target endpoint, so it can be streamed""" import json import yaml diff --git a/setup.md b/setup.md index d6ccda3..8835c4f 100644 --- a/setup.md +++ b/setup.md @@ -36,6 +36,9 @@ Set up a MongoDB database and set `MONGO_URI` to the MongoDB database connection - `PROXY_USER` (optional) - `PROXY_PASS` (optional) +Want to use a proxy list? See the according section! +Keep in mind to set `USE_PROXY_LIST` to `True`! Otherwise, the proxy list won't be used. + ### `ACTUAL_IPS` (optional) This is a security measure to make sure a proxy, VPN, Tor or any other IP hiding service is used by the host when accessing "Closed"AI's API. It is a space separated list of IP addresses that are allowed to access the API. @@ -49,13 +52,36 @@ You can also just add the *beginning* of an API address, like `12.123.` (without `CORE_API_KEY` specifies the **very secret key** for which need to access the entire user database etc. `TEST_NOVA_KEY` is the API key the which is used in tests. It should be one with tons of credits. -## Webhooks +### Webhooks `DISCORD_WEBHOOK__USER_CREATED` is the Discord webhook URL for when a user is created. `DISCORD_WEBHOOK__API_ISSUE` is the Discord webhook URL for when an API issue occurs. -## Other +### Other `KEYGEN_INFIX` can be almost any string (avoid spaces or special characters) - this string will be put in the middle of every NovaAI API key which is generated. This is useful for identifying the source of the key using e.g. RegEx. +## Proxy Lists +To use proxy lists, navigate to `api/secret/proxies/` and create the following files: +- `http.txt` +- `socks4.txt` +- `socks5.txt` + +Then, paste your proxies in the following format: + +``` +[username:password@]host:port +``` + +e.g. + +``` +1.2.3.4:8080 +user:pass@127.0.0.1:1337 +``` + +You can use comments just like in Python. + +**Important:** to use the proxy lists, you need to change the `USE_PROXY_LIST` environment variable to `True`! + ## Run > **Warning:** read the according section for production usage!