From d741a43b6bc263ce425b57bce36aac6df2cced72 Mon Sep 17 00:00:00 2001 From: Game_Time <108236317+RayBytes@users.noreply.github.com> Date: Thu, 17 Aug 2023 16:11:35 +0500 Subject: [PATCH 1/5] Re-added rate limiting, fixed a bunch of issues --- api/config/config.yml | 59 ++++++++++++++++++++++++++++++++++++++++++ api/config/credits.yml | 19 -------------- api/db/stats.py | 7 ++--- api/db/users.py | 2 +- api/streaming.py | 24 ++++++++--------- api/transfer.py | 9 ++++--- rewards/autocredits.py | 21 +++++---------- tests/__main__.py | 11 +------- 8 files changed, 89 insertions(+), 63 deletions(-) create mode 100644 api/config/config.yml delete mode 100644 api/config/credits.yml diff --git a/api/config/config.yml b/api/config/config.yml new file mode 100644 index 0000000..593392c --- /dev/null +++ b/api/config/config.yml @@ -0,0 +1,59 @@ +max-credits: 100001 +max-credits-owner: 694201337 +start-credits: 1000 + +costs: + other: 10 + + chat-models: + gpt-3: 10 + gpt-4: 30 + gpt-4-32k: 100 + +## Roles Explanation + +# Bonuses: They are a multiplier for costs +# They work like: final_cost = cost * bonus +# Rate limits: Limit the requests of the user +# The rate limit is by how many seconds until a new request can be done. + +## TODO: Setup proper rate limit settings for each role +## Current settings are: +## **NOT MEANT FOR PRODUCTION. DO NOT USE WITH THESE SETTINGS.** + +roles: + owner: + bonus: 0.1 + rate_limit: + other: 60 + gpt-3: 60 + gpt-4: 35 + gpt-4-32k: 5 + admin: + bonus: 0.3 + rate_limit: + other: 60 + gpt-3: 60 + gpt-4: 30 + gpt-4-32k: 4 + helper: + bonus: 0.4 + rate_limit: + other: 60 + gpt-3: 60 + gpt-4: 25 + gpt-4-32k: 3 + booster: + bonus: 0.5 + rate_limit: + other: 60 + gpt-3: 60 + gpt-4: 20 + gpt-4-32k: 2 + default: + bonus: 0 + rate_limit: + other: 60 + gpt-3: 60 + gpt-4: 15 + gpt-4-32k: 1 \ No newline at end of file diff --git a/api/config/credits.yml b/api/config/credits.yml deleted file mode 100644 index 29eeaa9..0000000 --- a/api/config/credits.yml +++ /dev/null @@ -1,19 +0,0 @@ -max-credits: 100001 -max-credits-owner: 694201337 -start-credits: 1000 - -costs: - other: 10 - - chat-models: - gpt-3: 10 - gpt-4: 30 - gpt-4-32k: 100 - -# bonuses are multiplier for costs: -# final_cost = cost * bonus -bonuses: - owner: 0.1 - admin: 0.3 - helper: 0.4 - booster: 0.5 diff --git a/api/db/stats.py b/api/db/stats.py index b2edeee..4fe7c56 100644 --- a/api/db/stats.py +++ b/api/db/stats.py @@ -26,7 +26,7 @@ class StatsManager: self.conn = AsyncIOMotorClient(os.getenv('MONGO_URI')) async def _get_collection(self, collection_name: str): - return self.conn[os.getenv('MONGO_NAME', 'nova-test')][collection_name] + return self.conn['nova-core'][collection_name] async def add_date(self): date = datetime.datetime.now(pytz.timezone('GMT')).strftime('%Y.%m.%d') @@ -62,5 +62,6 @@ class StatsManager: return await db.find_one({obj_filter}) if __name__ == '__main__': - asyncio.run(Stats.add_date()) - asyncio.run(Stats.add_path('/__demo/test')) + stats = StatsManager() + asyncio.run(stats.add_date()) + asyncio.run(stats.add_path('/__demo/test')) diff --git a/api/db/users.py b/api/db/users.py index 056c399..63073d5 100644 --- a/api/db/users.py +++ b/api/db/users.py @@ -9,7 +9,7 @@ from motor.motor_asyncio import AsyncIOMotorClient load_dotenv() -with open('config/credits.yml', encoding='utf8') as f: +with open('config/config.yml', encoding='utf8') as f: credits_config = yaml.safe_load(f) ## MONGODB Setup diff --git a/api/streaming.py b/api/streaming.py index 5ed64cf..4ff7ecc 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -26,12 +26,12 @@ import yaml load_dotenv() ## Loads config which contains rate limits -with open('config/credits.yml', encoding='utf8') as f: +with open('config/config.yml', encoding='utf8') as f: config = yaml.safe_load(f) ## Where all rate limit requested data will be stored. # Rate limit data is **not persistent** (It will be deleted on server stop/restart). -# user_last_request_time = {} +user_last_request_time = {} DEMO_PAYLOAD = { 'model': 'gpt-3.5-turbo', @@ -93,18 +93,18 @@ async def stream( ## Rate limits user. # If rate limit is exceeded, error code 429. Otherwise, lets the user pass but notes down # last request time for future requests. - # if user: - # role = user.get('role', 'default') - # rate_limit = config['roles'][role]['rate_limit'].get(payload['model'], 10) + if user: + role = user.get('role', 'default') + rate_limit = config['roles'].get(role, 1)['rate_limit'].get(payload['model'], 1) - # last_request_time = user_last_request_time.get(user['api_key']) - # time_since_last_request = datetime.datetime.now() - last_request_time + last_request_time = user_last_request_time.get(user['api_key']) + time_since_last_request = datetime.now() - last_request_time - # if time_since_last_request < datetime.timedelta(seconds=rate_limit): - # yield await errors.yield_error(429, 'Rate limit exceeded', "You are making requests too quickly. Please wait and try again later. Ask a administrator if you think this shouldn't happen.") - # return - # else: - # user_last_request_time[user['_id']] = datetime.datetime.now() + if time_since_last_request < datetime.timedelta(seconds=rate_limit): + yield await errors.yield_error(429, "Rate limit exceeded', 'You are making requests too quickly. Please wait and try again later. Ask a administrator if you think this shouldn't happen. ") + return + else: + user_last_request_time[user['_id']] = datetime.now() ## Setup managers db = UserManager() diff --git a/api/transfer.py b/api/transfer.py index 77e586f..98c3093 100644 --- a/api/transfer.py +++ b/api/transfer.py @@ -16,8 +16,8 @@ load_dotenv() models_list = json.load(open('models.json')) -with open('config/credits.yml', encoding='utf8') as f: - credits_config = yaml.safe_load(f) +with open('config/config.yml', encoding='utf8') as f: + config = yaml.safe_load(f) async def handle(incoming_request): """ @@ -62,7 +62,7 @@ async def handle(incoming_request): if path_contains_models: return fastapi.responses.JSONResponse(content=models_list) - costs = credits_config['costs'] + costs = config['costs'] cost = costs['other'] if 'chat/completions' in path: @@ -77,7 +77,8 @@ async def handle(incoming_request): if policy_violation: return await errors.error(400, f'The request contains content which violates this model\'s policies for "{policy_violation}".', 'We currently don\'t support any NSFW models.') - role_cost_multiplier = credits_config['bonuses'].get(user['role'], 1) + role = user.get('role', 'default') + role_cost_multiplier = config['roles'].get(role, 1)['bonus'] cost = round(cost * role_cost_multiplier) if user['credits'] < cost: diff --git a/rewards/autocredits.py b/rewards/autocredits.py index f7ea815..8aee8ff 100644 --- a/rewards/autocredits.py +++ b/rewards/autocredits.py @@ -1,15 +1,12 @@ -import os +import sys +# Weird hack because PYTHON IS GOOD LANGUAGE :)))) +sys.path.append('../') +from api.db.users import UserManager -from dotenv import load_dotenv - -load_dotenv() - -async def get_all_users(client): - users = client[os.getenv('MONGO_NAME', 'nova-test')]['users'] - return users async def update_credits(pymongo_client, settings=None): - users = await get_all_users(pymongo_client) + manager = UserManager() + users = await manager.get_all_users(pymongo_client) if not settings: users.update_many({}, {'$inc': {'credits': 2500}}) @@ -17,8 +14,4 @@ async def update_credits(pymongo_client, settings=None): else: for key, value in settings.items(): users.update_many( - {'level': key}, - {'$inc': - {'credits': int(value)} - } - ) + {'level': key}, {'$inc': {'credits': int(value)}}) diff --git a/tests/__main__.py b/tests/__main__.py index 500ae92..402b966 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -10,16 +10,7 @@ from dotenv import load_dotenv load_dotenv() MODEL = 'gpt-3.5-turbo' -# MESSAGES = [ -# { -# 'role': 'system', -# 'content': 'Always answer with "3", no matter what the user asks for. No exceptions. Just answer with the number "3". Nothing else. Just "3". No punctuation.' -# }, -# { -# 'role': 'user', -# 'content': '1+1=', -# }, -# ] + MESSAGES = [ { 'role': 'user', From b09bf688b43f9707ce6bd67031aeee835e5dd457 Mon Sep 17 00:00:00 2001 From: Game_Time <108236317+RayBytes@users.noreply.github.com> Date: Thu, 17 Aug 2023 16:13:20 +0500 Subject: [PATCH 2/5] Update stats.py with env mongodb --- api/db/stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/db/stats.py b/api/db/stats.py index 4fe7c56..1317a05 100644 --- a/api/db/stats.py +++ b/api/db/stats.py @@ -26,7 +26,7 @@ class StatsManager: self.conn = AsyncIOMotorClient(os.getenv('MONGO_URI')) async def _get_collection(self, collection_name: str): - return self.conn['nova-core'][collection_name] + return self.conn[os.getenv('MONGO_NAME', 'nova-test')][collection_name] async def add_date(self): date = datetime.datetime.now(pytz.timezone('GMT')).strftime('%Y.%m.%d') From 9e47dd0c4b729da6c85d065f01dc5a0ea464fa20 Mon Sep 17 00:00:00 2001 From: Game_Time <108236317+RayBytes@users.noreply.github.com> Date: Thu, 17 Aug 2023 16:29:50 +0500 Subject: [PATCH 3/5] Update tests.yml to make it WORK --- .github/workflows/tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 80310ba..f35803d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,8 +19,11 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt - - - name: Start API server + + - name: Start API server & set ENV + env: + MONGO_URI: ${{ secrets.MONGO_URI }} + TEST_NOVA_KEY: ${{ secrets.NOVA_KEY }} run: | python run From fb6dca5f7a4f94bcbf5f84d83c8cd94000f3acaa Mon Sep 17 00:00:00 2001 From: Game_Time <108236317+RayBytes@users.noreply.github.com> Date: Thu, 17 Aug 2023 16:45:36 +0500 Subject: [PATCH 4/5] Update tests.yml --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f35803d..b884031 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.x + python-version: 3.10 - name: Install dependencies run: | From 24e3f9f2fd77525fffd12634a89ab77363fd9e25 Mon Sep 17 00:00:00 2001 From: Game_Time <108236317+RayBytes@users.noreply.github.com> Date: Thu, 17 Aug 2023 16:48:05 +0500 Subject: [PATCH 5/5] Closes #8 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b884031..ccd7e5d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - - name: Start API server & set ENV + - name: Start API server env: MONGO_URI: ${{ secrets.MONGO_URI }} TEST_NOVA_KEY: ${{ secrets.NOVA_KEY }}