From d3466aa604879c4654166b591fa95c0f415ed23d Mon Sep 17 00:00:00 2001 From: Jun Siang Cheah Date: Thu, 30 Jan 2025 21:07:59 +0800 Subject: [PATCH] vertex: update available experimental models --- README.md | 4 ++-- src/pull_available_models.py | 31 +++++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d1370d2..fe2289e 100644 --- a/README.md +++ b/README.md @@ -258,10 +258,10 @@ This lists various services that provide free access or credits towards API-base Experimental Gemini model.
10 requests/minute - Gemini Flash Experimental + Gemini 2.0 Flash Thinking Experimental - Gemini Pro Experimental + Gemini 2.0 Experimental SambaNova Cloud$5 for 3 monthsLlama 3.1 8B30 requests/minute Llama 3.1 70B20 requests/minute Llama 3.1 405B10 requests/minute diff --git a/src/pull_available_models.py b/src/pull_available_models.py index 393a2cb..90f2075 100644 --- a/src/pull_available_models.py +++ b/src/pull_available_models.py @@ -10,11 +10,16 @@ from dotenv import load_dotenv from google.cloud import cloudquotas_v1 from mistralai import Mistral from concurrent.futures import ThreadPoolExecutor +import time load_dotenv() script_dir = os.path.dirname(os.path.abspath(__file__)) +# Global clients +mistral_client = Mistral(api_key=os.environ["MISTRAL_API_KEY"]) +last_mistral_request_time = 0 + MODEL_TO_NAME_MAPPING = { "@cf/deepseek-ai/deepseek-math-7b-instruct": "Deepseek Math 7B Instruct", "@cf/defog/sqlcoder-7b-2": "SQLCoder 7B 2", @@ -620,6 +625,20 @@ def fetch_lambda_models(logger): return ret_models +def rate_limited_mistral_chat(client, **kwargs): + global last_mistral_request_time + + # Ensure at least 1 second between requests + current_time = time.time() + time_since_last = current_time - last_mistral_request_time + if time_since_last < 1: + time.sleep(1 - time_since_last) + + response = client.chat.complete(**kwargs) + last_mistral_request_time = time.time() + return response + + def fetch_samba_models(logger): logger.info("Fetching SambaNova models...") r = requests.get("https://community.sambanova.ai/t/rate-limits/321") @@ -630,9 +649,9 @@ Here is the web page to extract data from: {r.text} ``` """ - client = Mistral(api_key=os.environ["MISTRAL_API_KEY"]) logger.info("Extracting model rate limits from the provided web page...") - chat_response = client.chat.complete( + chat_response = rate_limited_mistral_chat( + mistral_client, model="mistral-large-latest", messages=[ { @@ -705,9 +724,9 @@ Here is the web page to extract data from: {body} ``` """ - client = Mistral(api_key=os.environ["MISTRAL_API_KEY"]) logger.info("Extracting model rate limits from the provided web page...") - chat_response = client.chat.complete( + chat_response = rate_limited_mistral_chat( + mistral_client, model="mistral-large-latest", messages=[ { @@ -997,10 +1016,10 @@ def main(): Experimental Gemini model.
10 requests/minute - Gemini Flash Experimental + Gemini 2.0 Flash Thinking Experimental - Gemini Pro Experimental + Gemini 2.0 Experimental """ table += ""