From 6f64b3dc6205bc71288a86a92434cbb393207572 Mon Sep 17 00:00:00 2001 From: Jun Siang Cheah Date: Sun, 25 Aug 2024 11:08:37 +0100 Subject: [PATCH] merge hyperbolic models from API --- README.md | 11 ++++-- src/pull_available_models.py | 71 +++++++++++++++++++++++++++++++----- 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9240e01..e091267 100644 --- a/README.md +++ b/README.md @@ -91,12 +91,15 @@ This lists various services that provide free access or credits towards API-base Dynamic Rate Limits.
Limited to models smaller than 10GB.
Some popular models are supported even if they exceed 10GB. Various open models - Hyperbolic (Free Testing Period)Llama 3 70B Instruct200 requests/minute -Hermes 3 Llama 3.1 70B200 requests/minute -Llama 3.1 70B Instruct200 requests/minute -Llama 3.1 405B Instruct200 requests/minute + Hyperbolic (Free Testing Period)Hermes 3 Llama 3.1 70B200 requests/minute +Hermes 3 Llama 3.1 70B (FP8)200 requests/minute +Llama 3 70B Instruct200 requests/minute Llama 3.1 405B Base (FP8)200 requests/minute +Llama 3.1 405B Instruct200 requests/minute +Llama 3.1 70B Instruct200 requests/minute Llama 3.1 8B Instruct200 requests/minute +Yi 1.5 34B Chat200 requests/minute +Yi 34B Chat200 requests/minute OVH AI Endpoints (Free Alpha)Token expires every 2 weeks.CodeLlama 13B Instruct12 requests/minute Llama 2 13B Chat12 requests/minute Llama 3 70B Instruct12 requests/minute diff --git a/src/pull_available_models.py b/src/pull_available_models.py index d1f2939..3c6042a 100644 --- a/src/pull_available_models.py +++ b/src/pull_available_models.py @@ -88,9 +88,20 @@ MODEL_TO_NAME_MAPPING = { "qwen/qwen2-72b-instruct": "Qwen 2 72B Instruct", "undi95/toppy-m-7b:free": "Toppy M 7B", "whisper-large-v3": "Whisper Large v3", + "01-ai/yi-34b-chat": "Yi 34B Chat", + "01-ai/yi-1.5-34b-chat": "Yi 1.5 34B Chat", + "nousresearch/hermes-3-llama-3.1-70b-fp8": "Hermes 3 Llama 3.1 70B (FP8)", } MISSING_MODELS = set() +HYPERBOLIC_IGNORED_MODELS = { + "Wifhat", + "FLUX.1-dev", + "StableDiffusion", + "Monad", + "TTS" +} + def get_model_name(id): id = id.lower() @@ -136,6 +147,7 @@ def get_groq_limits_for_model(model_id): json={ "model": model_id, "messages": [{"role": "user", "content": "Hi!"}], + "max_tokens": 1, "stream": True, }, stream=True, @@ -275,9 +287,21 @@ def fetch_ovh_models(): ret_models = sorted(ret_models, key=lambda x: x["name"]) return ret_models - def fetch_hyperbolic_models(): print("Fetching Hyperbolic models...") + firestore_models = fetch_hyperbolic_models_firestore() + api_models = fetch_hyperbolic_models_api() + for model in api_models: + if model["id"] not in [m["id"] for m in firestore_models]: + print(f"Adding model {model['id']} from API") + firestore_models.append(model) + for model in firestore_models: + if model["id"] not in [m["id"] for m in api_models]: + print(f"Warning: model {model['id']} from Firestore not in API") + return sorted(firestore_models, key=lambda x: x["name"]) + +def fetch_hyperbolic_models_firestore(): + print("Fetching Hyperbolic models from Firestore...") r = requests.post( "https://firestore.googleapis.com/v1/projects/ai-dashboard-cfd6a/databases/(default)/documents:runQuery", @@ -336,7 +360,7 @@ def fetch_hyperbolic_models(): ) r.raise_for_status() models = r.json() - print(f"Fetched {len(models)} models from Hyperbolic") + print(f"Fetched {len(models)} models from Hyperbolic's Firestore") ret_models = [] for model in models: model_data = model["document"]["fields"] @@ -356,6 +380,34 @@ def fetch_hyperbolic_models(): pprint(ret_models) return ret_models +def fetch_hyperbolic_models_api(): + print("Fetching Hyperbolic models from API...") + r = requests.get( + "https://api.hyperbolic.xyz/v1/models", + headers={ + "accept": "application/json", + "authorization": f"Bearer {os.environ['HYPERBOLIC_API_KEY']}", + }) + r.raise_for_status() + models = r.json()["data"] + print(f"Fetched {len(models)} models from Hyperbolic's API") + ret_models = [] + for model in models: + if model["id"] in HYPERBOLIC_IGNORED_MODELS: + print(f"Ignoring model {model['id']}") + continue + ret_models.append( + { + "id": model["id"], + "name": get_model_name(model["id"]), + "limits": { + "requests/minute": 200, + }, + } + ) + pprint(ret_models) + return ret_models + def fetch_gemini_limits(): print("Fetching Gemini limits...") client = cloudquotas_v1.CloudQuotasClient() @@ -382,7 +434,14 @@ def get_human_limits(model): def main(): - # markdown_table = "|Provider|Provider Limits/Notes|Model Name|Model Limits|\n|---|---|---|---|\n" + gemini_models = fetch_gemini_limits() + openrouter_models = fetch_openrouter_models() + hyperbolic_models = fetch_hyperbolic_models() + ovh_models = fetch_ovh_models() + cloudflare_models = fetch_cloudflare_models() + groq_models = fetch_groq_models() + + table = """ @@ -395,15 +454,12 @@ def main(): """ - groq_models = fetch_groq_models() for idx, model in enumerate(groq_models): table += f"{f'' if idx == 0 else ''}{ f'' if idx == 0 else ''}\n" - openrouter_models = fetch_openrouter_models() for idx, model in enumerate(openrouter_models): table += f"{f'' if idx == 0 else ''}{ f'' if idx == 0 else ''}\n" - gemini_models = fetch_gemini_limits() table += f""" @@ -464,15 +520,12 @@ def main(): """ - hyperbolic_models = fetch_hyperbolic_models() for idx, model in enumerate(hyperbolic_models): table += f"{f'' if idx == 0 else ''}{ f'' if idx == 0 else ''}\n" - ovh_models = fetch_ovh_models() for idx, model in enumerate(ovh_models): table += f"{f'' if idx == 0 else ''}{ f'' if idx == 0 else ''}\n" - cloudflare_models = fetch_cloudflare_models() for idx, model in enumerate(cloudflare_models): table += f"{f'' if idx == 0 else ''}{ f'' if idx == 0 else ''}\n"
Groq{model['name']}{get_human_limits(model)}
OpenRouter{model['name']}{get_human_limits(model)}
Google AI Studio Free tier Gemini API access not available within UK/CH/EEA/EU/
Data is used for training.
Hyperbolic (Free Testing Period){model['name']}{get_human_limits(model)}
OVH AI Endpoints (Free Alpha)Token expires every 2 weeks.{model['name']}{get_human_limits(model)}
Cloudflare Workers AI10000 neurons/day
Beta models have unlimited usage.
Typically 300 requests/min for text models.		{model['name']}