{"schema_version":1,"generated_at":"2026-11-22T18:23:52+00:00","run":{"id":"2026-11-v1","started_at":"2026-11-22T11:52:30+00:00","finished_at":"2026-11-22T17:37:03+00:00","code_hash":"4fb55c063ad2b644","settings":{"storage":{"root":"/Volumes/AI-Lab/indic-llms","hf_home":"/Volumes/AI-Lab/indic-llms/hf-cache","models_dir":"/Volumes/AI-Lab/indic-llms/models","datasets_dir":"/Volumes/AI-Lab/indic-llms/datasets"},"ollama":{"binary":"/usr/local/bin/ollama","host":"127.0.0.1:11435","models_dir":"/Volumes/AI-Lab/indic-llms/ollama","num_ctx":4096,"timeout_s":900,"profiles":{"throughput":{"OLLAMA_NUM_PARALLEL":"4","OLLAMA_MAX_LOADED_MODELS":"3","OLLAMA_FLASH_ATTENTION":"1","OLLAMA_KV_CACHE_TYPE":"q8_0"},"measure":{"OLLAMA_NUM_PARALLEL":"1","OLLAMA_MAX_LOADED_MODELS":"1","OLLAMA_FLASH_ATTENTION":"1","OLLAMA_KV_CACHE_TYPE":"q8_0"}},"request_concurrency":4,"import_from":"~/.ollama/models"},"site_data_dir":"/Users/suhasbhairav/Desktop/metriccoders/online-ai-tools/suhasbhairav-brand/indic-leaderboard-data","languages":["hi","kn","ta","te","ml","hinglish"],"sampling":{"seed":20261122,"mmlu_indic_per_language":100,"indic_mmlu_pro_per_language":60,"milu_per_language":100},"generation":{"temperature":0,"max_tokens_mcq":16,"max_tokens_short":160,"max_tokens_long":600,"api_max_tokens_floor":1024},"openrouter":{"base_url":"https://openrouter.ai/api/v1","env_key":"OPENROUTER_API_KEY","env_file":"../llm-verifications-workbenchmark/.env","concurrency":6,"timeout_s":180,"reasoning_effort":"low"},"judge":{"models":["gemma4-31b","qwen3-30b-a3b","gemma3-12b"],"spot_check_fraction":0.1},"self_host":{"machine":"Apple M3 Max, 64 GB","watts":90,"inr_per_kwh":8},"currency":{"usd_inr":88.5,"rate_date":"2026-11-22","rate_source":"Set manually in config/settings.yaml; update before each monthly run"}},"private_set":{"version":"v0-2026-11-22","review_status":"drafted-by-claude-pending-native-review","items_per_language":17},"currency":{"usd_inr":88.5,"rate_date":"2026-11-22","rate_source":"Set manually in config/settings.yaml; update before each monthly run"},"self_host":{"machine":"Apple M3 Max, 64 GB","watts":90,"inr_per_kwh":8},"judge":"gemma4-31b","pricing_fetched_at":null},"languages":[{"code":"hi","name":"Hindi","native":"हिन्दी"},{"code":"kn","name":"Kannada","native":"ಕನ್ನಡ"},{"code":"ta","name":"Tamil","native":"தமிழ்"},{"code":"te","name":"Telugu","native":"తెలుగు"},{"code":"ml","name":"Malayalam","native":"മലയാളം"},{"code":"hinglish","name":"Hinglish","native":"Hinglish"}],"tasks":[{"key":"rag_qa","label":"Retrieval Q&A","group":"product","source":"private","source_label":null,"license_note":null},{"key":"tool_call","label":"Tool calling","group":"product","source":"private","source_label":null,"license_note":null},{"key":"intent","label":"Support routing","group":"product","source":"private","source_label":null,"license_note":null},{"key":"support_reply","label":"Support reply","group":"product","source":"private","source_label":null,"license_note":null},{"key":"summary","label":"Summarization","group":"product","source":"private","source_label":null,"license_note":null},{"key":"mmlu_indic","label":"MMLU-Indic","group":"knowledge","source":"sarvamai/mmlu-indic","source_label":"MMLU-Indic (Sarvam)","license_note":"MIT"},{"key":"indic_mmlu_pro","label":"IndicMMLU-Pro","group":"knowledge","source":"LinguaLift/IndicMMLU-Pro","source_label":"IndicMMLU-Pro","license_note":"see dataset card (no license field)"},{"key":"milu","label":"MILU","group":"knowledge","source":"ai4bharat/MILU","source_label":"MILU (AI4Bharat)","license_note":"AI4Bharat terms, see dataset card"}],"models":[{"id":"gemma4-31b","slug":"gemma4-31b","name":"Gemma 4 31B","maker":"Google","origin":"global","hosting":"self-hosted","backend":"ollama","params_b":31,"active_params_b":null,"license":"Gemma","languages":null,"conflict":false,"version":{"ollama_tag":"gemma4:31b","digest":"92808281a855666acf35743d49004bb30c810152cd004baf78f31b4578920e49","quantization":"","parameter_size":"","family":"","format":"safetensors","source":"google/gemma-4-31b-it","runtime":"ollama 0.40.10","num_ctx":4096,"think":false,"min_max_tokens":null},"weights_gb":19.4,"peak_memory_gb":23.3,"price_usd_per_m":null,"overall":{"product":100,"knowledge":60.8,"ttft_s":1.8,"output_tps":10.5},"english":{"tokens_per_word":1.53},"errors":0,"by_lang":{"hi":{"product":100,"knowledge":61.4,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":76,"n":100,"ci95":[66.8,83.3]},"indic_mmlu_pro":{"score":46.7,"n":60,"ci95":[34.6,59.1]}},"fidelity":88.9,"tokens_per_word":1.54,"vs_english":1.15,"inr_per_1000_words":null,"seconds_per_1000_words":164.2,"inr_electricity_per_1000_words":0.0328,"inr_per_1000_tasks":null,"ttft_s":1.95,"output_tps":9.4,"items":177},"kn":{"product":100,"knowledge":55.9,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":70,"n":100,"ci95":[60.4,78.1]},"indic_mmlu_pro":{"score":41.7,"n":60,"ci95":[30.1,54.3]}},"fidelity":88.9,"tokens_per_word":3.14,"vs_english":1.59,"inr_per_1000_words":null,"seconds_per_1000_words":336.3,"inr_electricity_per_1000_words":0.0673,"inr_per_1000_tasks":null,"ttft_s":2,"output_tps":9.3,"items":177},"ta":{"product":100,"knowledge":47.4,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":58,"n":100,"ci95":[48.2,67.2]},"indic_mmlu_pro":{"score":36.7,"n":60,"ci95":[25.6,49.3]}},"fidelity":88.9,"tokens_per_word":2.46,"vs_english":1.3,"inr_per_1000_words":null,"seconds_per_1000_words":255.8,"inr_electricity_per_1000_words":0.0512,"inr_per_1000_tasks":null,"ttft_s":1.6,"output_tps":9.6,"items":177},"te":{"product":100,"knowledge":56,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":72,"n":100,"ci95":[62.5,79.9]},"indic_mmlu_pro":{"score":40,"n":60,"ci95":[28.6,52.6]}},"fidelity":88.9,"tokens_per_word":2.88,"vs_english":1.51,"inr_per_1000_words":null,"seconds_per_1000_words":275.1,"inr_electricity_per_1000_words":0.055,"inr_per_1000_tasks":null,"ttft_s":1.75,"output_tps":10.5,"items":177},"ml":{"product":100,"knowledge":69,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":69,"n":100,"ci95":[59.4,77.2]}},"fidelity":88.9,"tokens_per_word":3.15,"vs_english":1.6,"inr_per_1000_words":null,"seconds_per_1000_words":302.9,"inr_electricity_per_1000_words":0.0606,"inr_per_1000_tasks":null,"ttft_s":1.61,"output_tps":10.4,"items":117},"hinglish":{"product":100,"knowledge":75,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":75,"n":100,"ci95":[65.7,82.5]}},"fidelity":100,"tokens_per_word":1.57,"vs_english":1.09,"inr_per_1000_words":null,"seconds_per_1000_words":135.2,"inr_electricity_per_1000_words":0.027,"inr_per_1000_tasks":null,"ttft_s":1.47,"output_tps":11.6,"items":117}}},{"id":"gemma4-26b","slug":"gemma4-26b","name":"Gemma 4 26B A4B","maker":"Google","origin":"global","hosting":"self-hosted","backend":"ollama","params_b":26,"active_params_b":4,"license":"Gemma","languages":null,"conflict":false,"version":{"ollama_tag":"gemma4:26b","digest":"48eb98ec778c163d87f69d735fcdd79e345063af9af4685613fd018ae5debf5e","quantization":"","parameter_size":"","family":"","format":"safetensors","source":"google/gemma-4-26b-a4b-it","runtime":"ollama 0.40.10","num_ctx":4096,"think":false,"min_max_tokens":null},"weights_gb":16.1,"peak_memory_gb":17.5,"price_usd_per_m":null,"overall":{"product":99,"knowledge":52.8,"ttft_s":0.34,"output_tps":49.9},"english":{"tokens_per_word":1.53},"errors":0,"by_lang":{"hi":{"product":100,"knowledge":58.5,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":72,"n":100,"ci95":[62.5,79.9]},"indic_mmlu_pro":{"score":45,"n":60,"ci95":[33.1,57.5]}},"fidelity":88.9,"tokens_per_word":1.54,"vs_english":1.15,"inr_per_1000_words":null,"seconds_per_1000_words":32,"inr_electricity_per_1000_words":0.0064,"inr_per_1000_tasks":null,"ttft_s":0.52,"output_tps":48.1,"items":177},"kn":{"product":95,"knowledge":47.6,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":75,"n":1,"ci95":[11.8,98.5]},"mmlu_indic":{"score":52,"n":100,"ci95":[42.3,61.5]},"indic_mmlu_pro":{"score":43.3,"n":60,"ci95":[31.6,55.9]}},"fidelity":88.9,"tokens_per_word":3.14,"vs_english":1.59,"inr_per_1000_words":null,"seconds_per_1000_words":67,"inr_electricity_per_1000_words":0.0134,"inr_per_1000_tasks":null,"ttft_s":0.33,"output_tps":46.8,"items":177},"ta":{"product":100,"knowledge":44.5,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":44,"n":100,"ci95":[34.7,53.8]},"indic_mmlu_pro":{"score":45,"n":60,"ci95":[33.1,57.5]}},"fidelity":88.9,"tokens_per_word":2.46,"vs_english":1.3,"inr_per_1000_words":null,"seconds_per_1000_words":50.4,"inr_electricity_per_1000_words":0.0101,"inr_per_1000_tasks":null,"ttft_s":0.29,"output_tps":48.8,"items":177},"te":{"product":98.8,"knowledge":48.9,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":56,"n":100,"ci95":[46.2,65.3]},"indic_mmlu_pro":{"score":41.7,"n":60,"ci95":[30.1,54.3]}},"fidelity":88.9,"tokens_per_word":2.88,"vs_english":1.51,"inr_per_1000_words":null,"seconds_per_1000_words":59.5,"inr_electricity_per_1000_words":0.0119,"inr_per_1000_tasks":null,"ttft_s":0.32,"output_tps":48.4,"items":177},"ml":{"product":100,"knowledge":54,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":54,"n":100,"ci95":[44.3,63.4]}},"fidelity":88.9,"tokens_per_word":3.15,"vs_english":1.6,"inr_per_1000_words":null,"seconds_per_1000_words":62.9,"inr_electricity_per_1000_words":0.0126,"inr_per_1000_tasks":null,"ttft_s":0.35,"output_tps":50,"items":117},"hinglish":{"product":100,"knowledge":63,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":63,"n":100,"ci95":[53.2,71.8]}},"fidelity":100,"tokens_per_word":1.57,"vs_english":1.09,"inr_per_1000_words":null,"seconds_per_1000_words":31.1,"inr_electricity_per_1000_words":0.0062,"inr_per_1000_tasks":null,"ttft_s":0.3,"output_tps":50.5,"items":117}}},{"id":"gemma4-e4b","slug":"gemma4-e4b","name":"Gemma 4 E4B","maker":"Google","origin":"global","hosting":"self-hosted","backend":"ollama","params_b":4,"active_params_b":null,"license":"Gemma","languages":null,"conflict":false,"version":{"ollama_tag":"gemma4:e4b","digest":"eb2177adf7646d15faead94fdad6d3e170dc8015653418a3b7d5d52a6380be5b","quantization":"","parameter_size":"","family":"","format":"safetensors","source":null,"runtime":"ollama 0.40.10","num_ctx":4096,"think":false,"min_max_tokens":null},"weights_gb":9.5,"peak_memory_gb":9.6,"price_usd_per_m":null,"overall":{"product":98.8,"knowledge":42.1,"ttft_s":0.2,"output_tps":69.7},"english":{"tokens_per_word":1.53},"errors":0,"by_lang":{"hi":{"product":100,"knowledge":44.5,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":59,"n":100,"ci95":[49.2,68.1]},"indic_mmlu_pro":{"score":30,"n":60,"ci95":[19.9,42.5]}},"fidelity":88.9,"tokens_per_word":1.54,"vs_english":1.15,"inr_per_1000_words":null,"seconds_per_1000_words":23,"inr_electricity_per_1000_words":0.0046,"inr_per_1000_tasks":null,"ttft_s":0.16,"output_tps":67.1,"items":177},"kn":{"product":97.5,"knowledge":37.5,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":87.5,"n":1,"ci95":[15.9,99.6]},"mmlu_indic":{"score":45,"n":100,"ci95":[35.6,54.8]},"indic_mmlu_pro":{"score":30,"n":60,"ci95":[19.9,42.5]}},"fidelity":88.9,"tokens_per_word":3.14,"vs_english":1.59,"inr_per_1000_words":null,"seconds_per_1000_words":43.7,"inr_electricity_per_1000_words":0.0087,"inr_per_1000_tasks":null,"ttft_s":0.24,"output_tps":71.8,"items":177},"ta":{"product":100,"knowledge":39,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":43,"n":100,"ci95":[33.7,52.8]},"indic_mmlu_pro":{"score":35,"n":60,"ci95":[24.2,47.6]}},"fidelity":88.9,"tokens_per_word":2.46,"vs_english":1.3,"inr_per_1000_words":null,"seconds_per_1000_words":33.2,"inr_electricity_per_1000_words":0.0066,"inr_per_1000_tasks":null,"ttft_s":0.23,"output_tps":74.1,"items":177},"te":{"product":100,"knowledge":37.5,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":45,"n":100,"ci95":[35.6,54.8]},"indic_mmlu_pro":{"score":30,"n":60,"ci95":[19.9,42.5]}},"fidelity":88.9,"tokens_per_word":2.88,"vs_english":1.51,"inr_per_1000_words":null,"seconds_per_1000_words":35,"inr_electricity_per_1000_words":0.007,"inr_per_1000_tasks":null,"ttft_s":0.25,"output_tps":82.2,"items":177},"ml":{"product":100,"knowledge":45,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":45,"n":100,"ci95":[35.6,54.8]}},"fidelity":88.9,"tokens_per_word":3.15,"vs_english":1.6,"inr_per_1000_words":null,"seconds_per_1000_words":43.7,"inr_electricity_per_1000_words":0.0087,"inr_per_1000_tasks":null,"ttft_s":0.24,"output_tps":72,"items":117},"hinglish":{"product":95,"knowledge":49,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":75,"n":1,"ci95":[11.8,98.5]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":49,"n":100,"ci95":[39.4,58.7]}},"fidelity":100,"tokens_per_word":1.57,"vs_english":1.09,"inr_per_1000_words":null,"seconds_per_1000_words":22.5,"inr_electricity_per_1000_words":0.0045,"inr_per_1000_tasks":null,"ttft_s":0.22,"output_tps":69.7,"items":117}}},{"id":"sarvam-m","slug":"sarvam-m","name":"Sarvam-M 24B","maker":"Sarvam AI","origin":"indian","hosting":"self-hosted","backend":"ollama","params_b":24,"active_params_b":null,"license":"Apache-2.0","languages":null,"conflict":false,"version":{"ollama_tag":"hf.co/bartowski/sarvamai_sarvam-m-GGUF:Q4_K_M","digest":"e07889258592064038b868ae6ae1f3a5897d5cffe736cb67ac62e6f015521711","quantization":"Q4_K_M","parameter_size":"23.6B","family":"llama","format":"gguf","source":"sarvamai/sarvam-m","runtime":"ollama 0.40.10","num_ctx":4096,"think":false,"min_max_tokens":null},"weights_gb":14.3,"peak_memory_gb":14,"price_usd_per_m":null,"overall":{"product":97.3,"knowledge":43,"ttft_s":1.07,"output_tps":10.6},"english":{"tokens_per_word":1.5},"errors":0,"by_lang":{"hi":{"product":98.8,"knowledge":43.5,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":57,"n":100,"ci95":[47.2,66.3]},"indic_mmlu_pro":{"score":30,"n":60,"ci95":[19.9,42.5]}},"fidelity":87.5,"tokens_per_word":2.1,"vs_english":1.6,"inr_per_1000_words":null,"seconds_per_1000_words":208.1,"inr_electricity_per_1000_words":0.0416,"inr_per_1000_tasks":null,"ttft_s":1.08,"output_tps":10.1,"items":177},"kn":{"product":98.8,"knowledge":40.5,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"mmlu_indic":{"score":46,"n":100,"ci95":[36.6,55.7]},"indic_mmlu_pro":{"score":35,"n":60,"ci95":[24.2,47.6]}},"fidelity":80,"tokens_per_word":3.74,"vs_english":1.94,"inr_per_1000_words":null,"seconds_per_1000_words":364,"inr_electricity_per_1000_words":0.0728,"inr_per_1000_tasks":null,"ttft_s":1.25,"output_tps":10.3,"items":177},"ta":{"product":100,"knowledge":38.4,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":40,"n":100,"ci95":[30.9,49.8]},"indic_mmlu_pro":{"score":36.7,"n":60,"ci95":[25.6,49.3]}},"fidelity":80,"tokens_per_word":3.68,"vs_english":2,"inr_per_1000_words":null,"seconds_per_1000_words":358.2,"inr_electricity_per_1000_words":0.0716,"inr_per_1000_tasks":null,"ttft_s":1.28,"output_tps":10.3,"items":177},"te":{"product":87.5,"knowledge":40.6,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":75,"n":4,"ci95":[30.1,95.4]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":68.8,"n":1,"ci95":[10,97.8]},"summary":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"mmlu_indic":{"score":43,"n":100,"ci95":[33.7,52.8]},"indic_mmlu_pro":{"score":38.3,"n":60,"ci95":[27.1,51]}},"fidelity":80,"tokens_per_word":3.65,"vs_english":1.96,"inr_per_1000_words":null,"seconds_per_1000_words":345.5,"inr_electricity_per_1000_words":0.0691,"inr_per_1000_tasks":null,"ttft_s":1.21,"output_tps":10.6,"items":177},"ml":{"product":98.8,"knowledge":48,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":48,"n":100,"ci95":[38.5,57.7]}},"fidelity":80,"tokens_per_word":4.11,"vs_english":2.13,"inr_per_1000_words":null,"seconds_per_1000_words":376.4,"inr_electricity_per_1000_words":0.0753,"inr_per_1000_tasks":null,"ttft_s":1.35,"output_tps":10.9,"items":117},"hinglish":{"product":100,"knowledge":47,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":47,"n":100,"ci95":[37.5,56.7]}},"fidelity":100,"tokens_per_word":1.7,"vs_english":1.21,"inr_per_1000_words":null,"seconds_per_1000_words":155.4,"inr_electricity_per_1000_words":0.0311,"inr_per_1000_tasks":null,"ttft_s":0.92,"output_tps":10.9,"items":117}}},{"id":"gemma3-12b","slug":"gemma3-12b","name":"Gemma 3 12B","maker":"Google","origin":"global","hosting":"self-hosted","backend":"ollama","params_b":12,"active_params_b":null,"license":"Gemma","languages":null,"conflict":false,"version":{"ollama_tag":"gemma3:12b","digest":"b92b218582c521ba287f38f1d6057dcd3c6cbff036f13cac97b5d861f5e58ec3","quantization":"Q4_K_M","parameter_size":"11.8B","family":"gemma3","format":"gguf","source":"google/gemma-3-12b-it","runtime":"ollama 0.40.10","num_ctx":4096,"think":null,"min_max_tokens":null},"weights_gb":8.2,"peak_memory_gb":7,"price_usd_per_m":null,"overall":{"product":96.7,"knowledge":39.1,"ttft_s":0.59,"output_tps":21.1},"english":{"tokens_per_word":1.53},"errors":0,"by_lang":{"hi":{"product":100,"knowledge":38.4,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":55,"n":100,"ci95":[45.2,64.4]},"indic_mmlu_pro":{"score":21.7,"n":60,"ci95":[13.1,33.6]}},"fidelity":88.9,"tokens_per_word":1.54,"vs_english":1.15,"inr_per_1000_words":null,"seconds_per_1000_words":75.2,"inr_electricity_per_1000_words":0.015,"inr_per_1000_tasks":null,"ttft_s":0.54,"output_tps":20.5,"items":177},"kn":{"product":95.9,"knowledge":37.1,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":46,"n":100,"ci95":[36.6,55.7]},"indic_mmlu_pro":{"score":28.3,"n":60,"ci95":[18.5,40.8]}},"fidelity":88.9,"tokens_per_word":3.14,"vs_english":1.59,"inr_per_1000_words":null,"seconds_per_1000_words":152.2,"inr_electricity_per_1000_words":0.0304,"inr_per_1000_tasks":null,"ttft_s":0.65,"output_tps":20.6,"items":177},"ta":{"product":97.1,"knowledge":33.1,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":43,"n":100,"ci95":[33.7,52.8]},"indic_mmlu_pro":{"score":23.3,"n":60,"ci95":[14.4,35.4]}},"fidelity":88.9,"tokens_per_word":2.46,"vs_english":1.3,"inr_per_1000_words":null,"seconds_per_1000_words":117.3,"inr_electricity_per_1000_words":0.0235,"inr_per_1000_tasks":null,"ttft_s":0.61,"output_tps":21,"items":177},"te":{"product":100,"knowledge":39,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":48,"n":100,"ci95":[38.5,57.7]},"indic_mmlu_pro":{"score":30,"n":60,"ci95":[19.9,42.5]}},"fidelity":88.9,"tokens_per_word":2.88,"vs_english":1.51,"inr_per_1000_words":null,"seconds_per_1000_words":137.5,"inr_electricity_per_1000_words":0.0275,"inr_per_1000_tasks":null,"ttft_s":0.62,"output_tps":20.9,"items":177},"ml":{"product":95.9,"knowledge":41,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":41,"n":100,"ci95":[31.9,50.8]}},"fidelity":88.9,"tokens_per_word":3.15,"vs_english":1.6,"inr_per_1000_words":null,"seconds_per_1000_words":147.2,"inr_electricity_per_1000_words":0.0294,"inr_per_1000_tasks":null,"ttft_s":0.61,"output_tps":21.4,"items":117},"hinglish":{"product":91.2,"knowledge":46,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"summary":{"score":75,"n":1,"ci95":[11.8,98.5]},"mmlu_indic":{"score":46,"n":100,"ci95":[36.6,55.7]}},"fidelity":100,"tokens_per_word":1.57,"vs_english":1.09,"inr_per_1000_words":null,"seconds_per_1000_words":72.8,"inr_electricity_per_1000_words":0.0146,"inr_per_1000_tasks":null,"ttft_s":0.57,"output_tps":21.6,"items":117}}},{"id":"krutrim-2","slug":"krutrim-2","name":"Krutrim-2 12B Instruct","maker":"Krutrim (Ola)","origin":"indian","hosting":"self-hosted","backend":"ollama","params_b":12,"active_params_b":null,"license":"Krutrim Community","languages":null,"conflict":false,"version":{"ollama_tag":"hf.co/mradermacher/Krutrim-2-instruct-GGUF:Q4_K_M","digest":"5dcbc9fe1be3af23cebced235cbe794c0d7d2c79f68617faf53150f79332a6b1","quantization":"Q4_K_M","parameter_size":"12.2B","family":"llama","format":"gguf","source":"krutrim-ai-labs/Krutrim-2-instruct","runtime":"ollama 0.40.10","num_ctx":4096,"think":null,"min_max_tokens":null},"weights_gb":7.5,"peak_memory_gb":7,"price_usd_per_m":null,"overall":{"product":96.7,"knowledge":34.9,"ttft_s":0.56,"output_tps":20.9},"english":{"tokens_per_word":1.5},"errors":0,"by_lang":{"hi":{"product":98.8,"knowledge":32,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":44,"n":100,"ci95":[34.7,53.8]},"indic_mmlu_pro":{"score":20,"n":60,"ci95":[11.8,31.8]}},"fidelity":88.9,"tokens_per_word":2.1,"vs_english":1.6,"inr_per_1000_words":null,"seconds_per_1000_words":100.4,"inr_electricity_per_1000_words":0.0201,"inr_per_1000_tasks":null,"ttft_s":0.55,"output_tps":20.9,"items":177},"kn":{"product":96.2,"knowledge":31.6,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":40,"n":100,"ci95":[30.9,49.8]},"indic_mmlu_pro":{"score":23.3,"n":60,"ci95":[14.4,35.4]}},"fidelity":88.9,"tokens_per_word":3.74,"vs_english":1.94,"inr_per_1000_words":null,"seconds_per_1000_words":180.1,"inr_electricity_per_1000_words":0.036,"inr_per_1000_tasks":null,"ttft_s":0.62,"output_tps":20.8,"items":177},"ta":{"product":95,"knowledge":33.1,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":75,"n":1,"ci95":[11.8,98.5]},"mmlu_indic":{"score":43,"n":100,"ci95":[33.7,52.8]},"indic_mmlu_pro":{"score":23.3,"n":60,"ci95":[14.4,35.4]}},"fidelity":87.5,"tokens_per_word":3.68,"vs_english":2,"inr_per_1000_words":null,"seconds_per_1000_words":180.8,"inr_electricity_per_1000_words":0.0362,"inr_per_1000_tasks":null,"ttft_s":0.71,"output_tps":20.3,"items":177},"te":{"product":95,"knowledge":29.9,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":38,"n":100,"ci95":[29.1,47.8]},"indic_mmlu_pro":{"score":21.7,"n":60,"ci95":[13.1,33.6]}},"fidelity":88.9,"tokens_per_word":3.65,"vs_english":1.96,"inr_per_1000_words":null,"seconds_per_1000_words":183.1,"inr_electricity_per_1000_words":0.0366,"inr_per_1000_tasks":null,"ttft_s":0.63,"output_tps":19.9,"items":177},"ml":{"product":98.8,"knowledge":39,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":39,"n":100,"ci95":[30,48.8]}},"fidelity":88.9,"tokens_per_word":4.11,"vs_english":2.13,"inr_per_1000_words":null,"seconds_per_1000_words":199.3,"inr_electricity_per_1000_words":0.0399,"inr_per_1000_tasks":null,"ttft_s":0.71,"output_tps":20.6,"items":117},"hinglish":{"product":96.2,"knowledge":44,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":44,"n":100,"ci95":[34.7,53.8]}},"fidelity":100,"tokens_per_word":1.7,"vs_english":1.21,"inr_per_1000_words":null,"seconds_per_1000_words":81.3,"inr_electricity_per_1000_words":0.0163,"inr_per_1000_tasks":null,"ttft_s":0.51,"output_tps":20.9,"items":117}}},{"id":"aya-expanse-8b","slug":"aya-expanse-8b","name":"Aya Expanse 8B","maker":"Cohere","origin":"global","hosting":"self-hosted","backend":"ollama","params_b":8,"active_params_b":null,"license":"CC-BY-NC-4.0","languages":["hi","hinglish"],"conflict":false,"version":{"ollama_tag":"aya-expanse:8b","digest":"65f986688a01b456158c57b042bc48afcb85d060646a82d491d1c0a01375b10e","quantization":"Q4_K_M","parameter_size":"8.0B","family":"command-r","format":"gguf","source":"CohereLabs/aya-expanse-8b","runtime":"ollama 0.40.10","num_ctx":4096,"think":null,"min_max_tokens":null},"weights_gb":5.1,"peak_memory_gb":5.5,"price_usd_per_m":null,"overall":{"product":93.5,"knowledge":36,"ttft_s":0.54,"output_tps":23.9},"english":{"tokens_per_word":1.52},"errors":0,"by_lang":{"hi":{"product":97.1,"knowledge":35,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":100,"n":1,"ci95":[20.7,100]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":45,"n":100,"ci95":[35.6,54.8]},"indic_mmlu_pro":{"score":25,"n":60,"ci95":[15.8,37.2]}},"fidelity":88.9,"tokens_per_word":3.26,"vs_english":2.45,"inr_per_1000_words":null,"seconds_per_1000_words":137.3,"inr_electricity_per_1000_words":0.0275,"inr_per_1000_tasks":null,"ttft_s":0.65,"output_tps":23.7,"items":177},"hinglish":{"product":90,"knowledge":37,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":75,"n":1,"ci95":[11.8,98.5]},"summary":{"score":75,"n":1,"ci95":[11.8,98.5]},"mmlu_indic":{"score":37,"n":100,"ci95":[28.2,46.8]}},"fidelity":55.6,"tokens_per_word":1.71,"vs_english":1.2,"inr_per_1000_words":null,"seconds_per_1000_words":70.9,"inr_electricity_per_1000_words":0.0142,"inr_per_1000_tasks":null,"ttft_s":0.41,"output_tps":24.2,"items":117}}},{"id":"llama3.1-8b","slug":"llama3.1-8b","name":"Llama 3.1 8B Instruct","maker":"Meta","origin":"global","hosting":"self-hosted","backend":"ollama","params_b":8,"active_params_b":null,"license":"Llama 3.1 Community","languages":null,"conflict":false,"version":{"ollama_tag":"llama3.1:8b","digest":"46e0c10c039e019119339687c3c1757cc81b9da49709a3b3924863ba87ca666e","quantization":"Q4_K_M","parameter_size":"8.0B","family":"llama","format":"gguf","source":"meta-llama/Llama-3.1-8B-Instruct","runtime":"ollama 0.40.10","num_ctx":4096,"think":null,"min_max_tokens":null},"weights_gb":4.9,"peak_memory_gb":5,"price_usd_per_m":null,"overall":{"product":89.3,"knowledge":29.9,"ttft_s":0.71,"output_tps":27.5},"english":{"tokens_per_word":1.39},"errors":0,"by_lang":{"hi":{"product":95.9,"knowledge":31.5,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":43,"n":100,"ci95":[33.7,52.8]},"indic_mmlu_pro":{"score":20,"n":60,"ci95":[11.8,31.8]}},"fidelity":100,"tokens_per_word":2.51,"vs_english":2.06,"inr_per_1000_words":null,"seconds_per_1000_words":87.3,"inr_electricity_per_1000_words":0.0175,"inr_per_1000_tasks":null,"ttft_s":0.41,"output_tps":28.8,"items":177},"kn":{"product":89.6,"knowledge":24,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":28,"n":100,"ci95":[20.1,37.5]},"indic_mmlu_pro":{"score":20,"n":60,"ci95":[11.8,31.8]}},"fidelity":100,"tokens_per_word":12.45,"vs_english":6.96,"inr_per_1000_words":null,"seconds_per_1000_words":441.9,"inr_electricity_per_1000_words":0.0884,"inr_per_1000_tasks":null,"ttft_s":1.1,"output_tps":28.2,"items":177},"ta":{"product":88.4,"knowledge":23.6,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":68.8,"n":1,"ci95":[10,97.8]},"summary":{"score":87.5,"n":1,"ci95":[15.9,99.6]},"mmlu_indic":{"score":29,"n":100,"ci95":[21,38.5]},"indic_mmlu_pro":{"score":18.3,"n":60,"ci95":[10.6,29.9]}},"fidelity":100,"tokens_per_word":10.68,"vs_english":6.25,"inr_per_1000_words":null,"seconds_per_1000_words":389.6,"inr_electricity_per_1000_words":0.0779,"inr_per_1000_tasks":null,"ttft_s":1.05,"output_tps":27.4,"items":177},"te":{"product":87.1,"knowledge":25.4,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":68.8,"n":1,"ci95":[10,97.8]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":29,"n":100,"ci95":[21,38.5]},"indic_mmlu_pro":{"score":21.7,"n":60,"ci95":[13.1,33.6]}},"fidelity":100,"tokens_per_word":11.8,"vs_english":6.81,"inr_per_1000_words":null,"seconds_per_1000_words":442.4,"inr_electricity_per_1000_words":0.0885,"inr_per_1000_tasks":null,"ttft_s":1.09,"output_tps":26.7,"items":177},"ml":{"product":83.4,"knowledge":33,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":50,"n":1,"ci95":[5.5,94.5]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":33,"n":100,"ci95":[24.6,42.7]}},"fidelity":100,"tokens_per_word":12.36,"vs_english":6.91,"inr_per_1000_words":null,"seconds_per_1000_words":464.4,"inr_electricity_per_1000_words":0.0929,"inr_per_1000_tasks":null,"ttft_s":1.17,"output_tps":26.6,"items":117},"hinglish":{"product":91.2,"knowledge":42,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"summary":{"score":75,"n":1,"ci95":[11.8,98.5]},"mmlu_indic":{"score":42,"n":100,"ci95":[32.8,51.8]}},"fidelity":100,"tokens_per_word":1.64,"vs_english":1.26,"inr_per_1000_words":null,"seconds_per_1000_words":60.6,"inr_electricity_per_1000_words":0.0121,"inr_per_1000_tasks":null,"ttft_s":0.39,"output_tps":27.1,"items":117}}},{"id":"gemma3-4b","slug":"gemma3-4b","name":"Gemma 3 4B","maker":"Google","origin":"global","hosting":"self-hosted","backend":"ollama","params_b":4,"active_params_b":null,"license":"Gemma","languages":null,"conflict":false,"version":{"ollama_tag":"gemma3:4b","digest":"68ceab7e468a31c988b44d5a4984a0e21f58c4f7158c6ce29c1804babac38df7","quantization":"Q4_K_M","parameter_size":"3.9B","family":"gemma3","format":"gguf","source":"google/gemma-3-4b-it","runtime":"ollama 0.40.10","num_ctx":4096,"think":null,"min_max_tokens":null},"weights_gb":3.4,"peak_memory_gb":2.2,"price_usd_per_m":null,"overall":{"product":88.3,"knowledge":31,"ttft_s":0.18,"output_tps":57.1},"english":{"tokens_per_word":1.53},"errors":0,"by_lang":{"hi":{"product":88.7,"knowledge":34.4,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":75,"n":4,"ci95":[30.1,95.4]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"summary":{"score":87.5,"n":1,"ci95":[15.9,99.6]},"mmlu_indic":{"score":47,"n":100,"ci95":[37.5,56.7]},"indic_mmlu_pro":{"score":21.7,"n":60,"ci95":[13.1,33.6]}},"fidelity":88.9,"tokens_per_word":1.54,"vs_english":1.15,"inr_per_1000_words":null,"seconds_per_1000_words":27.3,"inr_electricity_per_1000_words":0.0055,"inr_per_1000_tasks":null,"ttft_s":0.16,"output_tps":56.5,"items":177},"kn":{"product":91.3,"knowledge":23.9,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":87.5,"n":1,"ci95":[15.9,99.6]},"summary":{"score":68.8,"n":1,"ci95":[10,97.8]},"mmlu_indic":{"score":31,"n":100,"ci95":[22.8,40.6]},"indic_mmlu_pro":{"score":16.7,"n":60,"ci95":[9.3,28]}},"fidelity":85.7,"tokens_per_word":3.14,"vs_english":1.59,"inr_per_1000_words":null,"seconds_per_1000_words":54.3,"inr_electricity_per_1000_words":0.0109,"inr_per_1000_tasks":null,"ttft_s":0.2,"output_tps":57.7,"items":177},"ta":{"product":91.2,"knowledge":26.4,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":75,"n":1,"ci95":[11.8,98.5]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":36,"n":100,"ci95":[27.3,45.8]},"indic_mmlu_pro":{"score":16.7,"n":60,"ci95":[9.3,28]}},"fidelity":50,"tokens_per_word":2.46,"vs_english":1.3,"inr_per_1000_words":null,"seconds_per_1000_words":43.1,"inr_electricity_per_1000_words":0.0086,"inr_per_1000_tasks":null,"ttft_s":0.2,"output_tps":57,"items":177},"te":{"product":90,"knowledge":23.4,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"summary":{"score":68.8,"n":1,"ci95":[10,97.8]},"mmlu_indic":{"score":30,"n":100,"ci95":[21.9,39.6]},"indic_mmlu_pro":{"score":16.7,"n":60,"ci95":[9.3,28]}},"fidelity":87.5,"tokens_per_word":2.88,"vs_english":1.51,"inr_per_1000_words":null,"seconds_per_1000_words":50.2,"inr_electricity_per_1000_words":0.01,"inr_per_1000_tasks":null,"ttft_s":0.2,"output_tps":57.4,"items":177},"ml":{"product":78.8,"knowledge":36,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":75,"n":4,"ci95":[30.1,95.4]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":50,"n":1,"ci95":[5.5,94.5]},"summary":{"score":68.8,"n":1,"ci95":[10,97.8]},"mmlu_indic":{"score":36,"n":100,"ci95":[27.3,45.8]}},"fidelity":66.7,"tokens_per_word":3.15,"vs_english":1.6,"inr_per_1000_words":null,"seconds_per_1000_words":54.4,"inr_electricity_per_1000_words":0.0109,"inr_per_1000_tasks":null,"ttft_s":0.2,"output_tps":57.9,"items":117},"hinglish":{"product":90,"knowledge":42,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":75,"n":1,"ci95":[11.8,98.5]},"summary":{"score":75,"n":1,"ci95":[11.8,98.5]},"mmlu_indic":{"score":42,"n":100,"ci95":[32.8,51.8]}},"fidelity":100,"tokens_per_word":1.57,"vs_english":1.09,"inr_per_1000_words":null,"seconds_per_1000_words":26.1,"inr_electricity_per_1000_words":0.0052,"inr_per_1000_tasks":null,"ttft_s":0.18,"output_tps":60.2,"items":117}}},{"id":"qwen3-8b","slug":"qwen3-8b","name":"Qwen3 8B","maker":"Alibaba Qwen","origin":"global","hosting":"self-hosted","backend":"ollama","params_b":8,"active_params_b":null,"license":"Apache-2.0","languages":null,"conflict":false,"version":{"ollama_tag":"qwen3:8b","digest":"500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41","quantization":"Q4_K_M","parameter_size":"8.2B","family":"qwen3","format":"gguf","source":"Qwen/Qwen3-8B","runtime":"ollama 0.40.10","num_ctx":8192,"think":false,"min_max_tokens":null},"weights_gb":5.2,"peak_memory_gb":4.8,"price_usd_per_m":null,"overall":{"product":88.3,"knowledge":34.6,"ttft_s":0.6,"output_tps":36.5},"english":{"tokens_per_word":1.5},"errors":0,"by_lang":{"hi":{"product":89.6,"knowledge":35.5,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":56,"n":100,"ci95":[46.2,65.3]},"indic_mmlu_pro":{"score":15,"n":60,"ci95":[8.1,26.1]}},"fidelity":100,"tokens_per_word":4.5,"vs_english":3.43,"inr_per_1000_words":null,"seconds_per_1000_words":123.3,"inr_electricity_per_1000_words":0.0247,"inr_per_1000_tasks":null,"ttft_s":0.53,"output_tps":36.5,"items":177},"kn":{"product":90.9,"knowledge":33,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":68.8,"n":1,"ci95":[10,97.8]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":36,"n":100,"ci95":[27.3,45.8]},"indic_mmlu_pro":{"score":30,"n":60,"ci95":[19.9,42.5]}},"fidelity":100,"tokens_per_word":10.18,"vs_english":5.29,"inr_per_1000_words":null,"seconds_per_1000_words":280.1,"inr_electricity_per_1000_words":0.056,"inr_per_1000_tasks":null,"ttft_s":0.81,"output_tps":36.3,"items":177},"ta":{"product":92.1,"knowledge":32.9,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":39,"n":100,"ci95":[30,48.8]},"indic_mmlu_pro":{"score":26.7,"n":60,"ci95":[17.1,39]}},"fidelity":100,"tokens_per_word":8.9,"vs_english":4.84,"inr_per_1000_words":null,"seconds_per_1000_words":243.8,"inr_electricity_per_1000_words":0.0488,"inr_per_1000_tasks":null,"ttft_s":0.79,"output_tps":36.5,"items":177},"te":{"product":80.9,"knowledge":33.1,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":43.8,"n":1,"ci95":[4.2,93.2]},"summary":{"score":75,"n":1,"ci95":[11.8,98.5]},"mmlu_indic":{"score":43,"n":100,"ci95":[33.7,52.8]},"indic_mmlu_pro":{"score":23.3,"n":60,"ci95":[14.4,35.4]}},"fidelity":100,"tokens_per_word":10.37,"vs_english":5.57,"inr_per_1000_words":null,"seconds_per_1000_words":285.1,"inr_electricity_per_1000_words":0.057,"inr_per_1000_tasks":null,"ttft_s":0.84,"output_tps":36.4,"items":177},"ml":{"product":86.8,"knowledge":39,"tasks":{"rag_qa":{"score":71.4,"n":7,"ci95":[35.9,91.8]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":39,"n":100,"ci95":[30,48.8]}},"fidelity":100,"tokens_per_word":10.47,"vs_english":5.44,"inr_per_1000_words":null,"seconds_per_1000_words":286.1,"inr_electricity_per_1000_words":0.0572,"inr_per_1000_tasks":null,"ttft_s":0.85,"output_tps":36.6,"items":117},"hinglish":{"product":89.6,"knowledge":34,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":100,"n":4,"ci95":[51,100]},"support_reply":{"score":62.5,"n":1,"ci95":[8.3,96.8]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":34,"n":100,"ci95":[25.5,43.7]}},"fidelity":100,"tokens_per_word":1.75,"vs_english":1.25,"inr_per_1000_words":null,"seconds_per_1000_words":46.8,"inr_electricity_per_1000_words":0.0094,"inr_per_1000_tasks":null,"ttft_s":0.33,"output_tps":37.4,"items":117}}},{"id":"llama3.2-3b","slug":"llama3.2-3b","name":"Llama 3.2 3B Instruct","maker":"Meta","origin":"global","hosting":"self-hosted","backend":"ollama","params_b":3,"active_params_b":null,"license":"Llama 3.2 Community","languages":null,"conflict":false,"version":{"ollama_tag":"llama3.2:3b","digest":"a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72","quantization":"Q4_K_M","parameter_size":"3.2B","family":"llama","format":"gguf","source":"meta-llama/Llama-3.2-3B-Instruct","runtime":"ollama 0.40.10","num_ctx":4096,"think":null,"min_max_tokens":null},"weights_gb":2,"peak_memory_gb":2.3,"price_usd_per_m":null,"overall":{"product":73.7,"knowledge":26.7,"ttft_s":0.36,"output_tps":53},"english":{"tokens_per_word":1.39},"errors":0,"by_lang":{"hi":{"product":80.9,"knowledge":29.1,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":50,"n":4,"ci95":[15,85]},"support_reply":{"score":87.5,"n":1,"ci95":[15.9,99.6]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":35,"n":100,"ci95":[26.4,44.7]},"indic_mmlu_pro":{"score":23.3,"n":60,"ci95":[14.4,35.4]}},"fidelity":100,"tokens_per_word":2.51,"vs_english":2.06,"inr_per_1000_words":null,"seconds_per_1000_words":44.7,"inr_electricity_per_1000_words":0.0089,"inr_per_1000_tasks":null,"ttft_s":0.18,"output_tps":56.1,"items":177},"kn":{"product":68.4,"knowledge":21.9,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":75,"n":4,"ci95":[30.1,95.4]},"intent":{"score":50,"n":4,"ci95":[15,85]},"support_reply":{"score":50,"n":1,"ci95":[5.5,94.5]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":27,"n":100,"ci95":[19.3,36.4]},"indic_mmlu_pro":{"score":16.7,"n":60,"ci95":[9.3,28]}},"fidelity":100,"tokens_per_word":12.45,"vs_english":6.96,"inr_per_1000_words":null,"seconds_per_1000_words":238.6,"inr_electricity_per_1000_words":0.0477,"inr_per_1000_tasks":null,"ttft_s":0.53,"output_tps":52.2,"items":177},"ta":{"product":68.4,"knowledge":23.6,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":75,"n":4,"ci95":[30.1,95.4]},"intent":{"score":50,"n":4,"ci95":[15,85]},"support_reply":{"score":50,"n":1,"ci95":[5.5,94.5]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":29,"n":100,"ci95":[21,38.5]},"indic_mmlu_pro":{"score":18.3,"n":60,"ci95":[10.6,29.9]}},"fidelity":100,"tokens_per_word":10.68,"vs_english":6.25,"inr_per_1000_words":null,"seconds_per_1000_words":203.9,"inr_electricity_per_1000_words":0.0408,"inr_per_1000_tasks":null,"ttft_s":0.51,"output_tps":52.4,"items":177},"te":{"product":72.1,"knowledge":22.4,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":75,"n":4,"ci95":[30.1,95.4]},"intent":{"score":50,"n":4,"ci95":[15,85]},"support_reply":{"score":68.8,"n":1,"ci95":[10,97.8]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":28,"n":100,"ci95":[20.1,37.5]},"indic_mmlu_pro":{"score":16.7,"n":60,"ci95":[9.3,28]}},"fidelity":100,"tokens_per_word":11.8,"vs_english":6.81,"inr_per_1000_words":null,"seconds_per_1000_words":226,"inr_electricity_per_1000_words":0.0452,"inr_per_1000_tasks":null,"ttft_s":0.53,"output_tps":52.2,"items":177},"ml":{"product":70.9,"knowledge":25,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":75,"n":4,"ci95":[30.1,95.4]},"intent":{"score":50,"n":4,"ci95":[15,85]},"support_reply":{"score":62.5,"n":1,"ci95":[8.3,96.8]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":25,"n":100,"ci95":[17.5,34.3]}},"fidelity":100,"tokens_per_word":12.36,"vs_english":6.91,"inr_per_1000_words":null,"seconds_per_1000_words":241.4,"inr_electricity_per_1000_words":0.0483,"inr_per_1000_tasks":null,"ttft_s":0.54,"output_tps":51.2,"items":117},"hinglish":{"product":81.3,"knowledge":38,"tasks":{"rag_qa":{"score":100,"n":7,"ci95":[64.6,100]},"tool_call":{"score":100,"n":4,"ci95":[51,100]},"intent":{"score":50,"n":4,"ci95":[15,85]},"support_reply":{"score":62.5,"n":1,"ci95":[8.3,96.8]},"summary":{"score":93.8,"n":1,"ci95":[18.2,99.9]},"mmlu_indic":{"score":38,"n":100,"ci95":[29.1,47.8]}},"fidelity":100,"tokens_per_word":1.64,"vs_english":1.26,"inr_per_1000_words":null,"seconds_per_1000_words":30.2,"inr_electricity_per_1000_words":0.006,"inr_per_1000_tasks":null,"ttft_s":0.15,"output_tps":54.4,"items":117}}},{"id":"bharatgpt-3b","slug":"bharatgpt-3b","name":"BharatGPT 3B Indic","maker":"CoRover","origin":"indian","hosting":"self-hosted","backend":"ollama","params_b":3,"active_params_b":null,"license":"Llama 3.2 Community","languages":null,"conflict":false,"version":{"ollama_tag":"hf.co/mradermacher/BharatGPT-3B-Indic-GGUF:Q4_K_M","digest":"eafb2b2e93221d6a82aeb71312a02093f88b1c6350b2d11921cbd3ae9459d9cb","quantization":"Q4_K_M","parameter_size":"3.21B","family":"llama","format":"gguf","source":"CoRover/BharatGPT-3B-Indic","runtime":"ollama 0.40.10","num_ctx":4096,"think":null,"min_max_tokens":null},"weights_gb":2,"peak_memory_gb":2.3,"price_usd_per_m":null,"overall":{"product":50.1,"knowledge":19.2,"ttft_s":0.31,"output_tps":61.4},"english":{"tokens_per_word":1.39},"errors":0,"by_lang":{"hi":{"product":57.1,"knowledge":19.4,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":0,"n":4,"ci95":[0,49]},"intent":{"score":75,"n":4,"ci95":[30.1,95.4]},"support_reply":{"score":43.8,"n":1,"ci95":[4.2,93.2]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":27,"n":100,"ci95":[19.3,36.4]},"indic_mmlu_pro":{"score":11.7,"n":60,"ci95":[5.8,22.2]}},"fidelity":100,"tokens_per_word":2.51,"vs_english":2.06,"inr_per_1000_words":null,"seconds_per_1000_words":39,"inr_electricity_per_1000_words":0.0078,"inr_per_1000_tasks":null,"ttft_s":0.17,"output_tps":64.4,"items":177},"kn":{"product":46.8,"knowledge":8.8,"tasks":{"rag_qa":{"score":71.4,"n":7,"ci95":[35.9,91.8]},"tool_call":{"score":25,"n":4,"ci95":[4.6,69.9]},"intent":{"score":50,"n":4,"ci95":[15,85]},"support_reply":{"score":18.8,"n":1,"ci95":[0.8,86.2]},"summary":{"score":68.8,"n":1,"ci95":[10,97.8]},"mmlu_indic":{"score":16,"n":100,"ci95":[10.1,24.4]},"indic_mmlu_pro":{"score":1.7,"n":60,"ci95":[0.3,8.9]}},"fidelity":100,"tokens_per_word":12.45,"vs_english":6.96,"inr_per_1000_words":null,"seconds_per_1000_words":203.9,"inr_electricity_per_1000_words":0.0408,"inr_per_1000_tasks":null,"ttft_s":0.47,"output_tps":61,"items":177},"ta":{"product":36.8,"knowledge":8.5,"tasks":{"rag_qa":{"score":71.4,"n":7,"ci95":[35.9,91.8]},"tool_call":{"score":0,"n":4,"ci95":[0,49]},"intent":{"score":25,"n":4,"ci95":[4.6,69.9]},"support_reply":{"score":31.2,"n":1,"ci95":[2.2,90]},"summary":{"score":56.2,"n":1,"ci95":[6.8,95.8]},"mmlu_indic":{"score":17,"n":100,"ci95":[10.9,25.5]},"indic_mmlu_pro":{"score":0,"n":60,"ci95":[0,6]}},"fidelity":100,"tokens_per_word":10.68,"vs_english":6.25,"inr_per_1000_words":null,"seconds_per_1000_words":173.9,"inr_electricity_per_1000_words":0.0348,"inr_per_1000_tasks":null,"ttft_s":0.44,"output_tps":61.4,"items":177},"te":{"product":45.9,"knowledge":15.2,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":0,"n":4,"ci95":[0,49]},"intent":{"score":50,"n":4,"ci95":[15,85]},"support_reply":{"score":43.8,"n":1,"ci95":[4.2,93.2]},"summary":{"score":50,"n":1,"ci95":[5.5,94.5]},"mmlu_indic":{"score":27,"n":100,"ci95":[19.3,36.4]},"indic_mmlu_pro":{"score":3.3,"n":60,"ci95":[0.9,11.4]}},"fidelity":100,"tokens_per_word":11.8,"vs_english":6.81,"inr_per_1000_words":null,"seconds_per_1000_words":195.7,"inr_electricity_per_1000_words":0.0391,"inr_per_1000_tasks":null,"ttft_s":0.5,"output_tps":60.3,"items":177},"ml":{"product":55.5,"knowledge":25,"tasks":{"rag_qa":{"score":71.4,"n":7,"ci95":[35.9,91.8]},"tool_call":{"score":50,"n":4,"ci95":[15,85]},"intent":{"score":50,"n":4,"ci95":[15,85]},"support_reply":{"score":25,"n":1,"ci95":[1.5,88.2]},"summary":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"mmlu_indic":{"score":25,"n":100,"ci95":[17.5,34.3]}},"fidelity":100,"tokens_per_word":12.36,"vs_english":6.91,"inr_per_1000_words":null,"seconds_per_1000_words":210,"inr_electricity_per_1000_words":0.042,"inr_per_1000_tasks":null,"ttft_s":0.5,"output_tps":58.9,"items":117},"hinglish":{"product":58.4,"knowledge":38,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":50,"n":4,"ci95":[15,85]},"intent":{"score":75,"n":4,"ci95":[30.1,95.4]},"support_reply":{"score":25,"n":1,"ci95":[1.5,88.2]},"summary":{"score":56.2,"n":1,"ci95":[6.8,95.8]},"mmlu_indic":{"score":38,"n":100,"ci95":[29.1,47.8]}},"fidelity":100,"tokens_per_word":1.64,"vs_english":1.26,"inr_per_1000_words":null,"seconds_per_1000_words":27.8,"inr_electricity_per_1000_words":0.0056,"inr_per_1000_tasks":null,"ttft_s":0.16,"output_tps":59.1,"items":117}}},{"id":"nemotron-4-mini-hindi-4b","slug":"nemotron-4-mini-hindi-4b","name":"Nemotron-4 Mini Hindi 4B","maker":"NVIDIA","origin":"global","hosting":"self-hosted","backend":"ollama","params_b":4,"active_params_b":null,"license":"NVIDIA Open Model","languages":["hi","hinglish"],"conflict":false,"version":{"ollama_tag":"hf.co/Mungert/Nemotron-4-Mini-Hindi-4B-Instruct-GGUF:Q4_K_M","digest":"f26ffa75cdc2a2788be478cb4547e22355dea62cfb34723125781947d4e7a0e7","quantization":"Q4_K_M","parameter_size":"4.19B","family":"nemotron","format":"gguf","source":"nvidia/Nemotron-4-Mini-Hindi-4B-Instruct","runtime":"ollama 0.40.10","num_ctx":4096,"think":null,"min_max_tokens":null},"weights_gb":2.9,"peak_memory_gb":1.9,"price_usd_per_m":null,"overall":{"product":49.8,"knowledge":35.5,"ttft_s":0.14,"output_tps":52.3},"english":{"tokens_per_word":1.53},"errors":0,"by_lang":{"hi":{"product":61.4,"knowledge":26.1,"tasks":{"rag_qa":{"score":57.1,"n":7,"ci95":[25,84.2]},"tool_call":{"score":75,"n":4,"ci95":[30.1,95.4]},"intent":{"score":25,"n":4,"ci95":[4.6,69.9]},"support_reply":{"score":75,"n":1,"ci95":[11.8,98.5]},"summary":{"score":75,"n":1,"ci95":[11.8,98.5]},"mmlu_indic":{"score":39,"n":100,"ci95":[30,48.8]},"indic_mmlu_pro":{"score":13.3,"n":60,"ci95":[6.9,24.2]}},"fidelity":66.7,"tokens_per_word":1.93,"vs_english":1.44,"inr_per_1000_words":null,"seconds_per_1000_words":36.1,"inr_electricity_per_1000_words":0.0072,"inr_per_1000_tasks":null,"ttft_s":0.17,"output_tps":53.4,"items":177},"hinglish":{"product":38.2,"knowledge":45,"tasks":{"rag_qa":{"score":28.6,"n":7,"ci95":[8.2,64.1]},"tool_call":{"score":75,"n":4,"ci95":[30.1,95.4]},"intent":{"score":0,"n":4,"ci95":[0,49]},"support_reply":{"score":31.2,"n":1,"ci95":[2.2,90]},"summary":{"score":56.2,"n":1,"ci95":[6.8,95.8]},"mmlu_indic":{"score":45,"n":100,"ci95":[35.6,54.8]}},"fidelity":100,"tokens_per_word":1.69,"vs_english":1.17,"inr_per_1000_words":null,"seconds_per_1000_words":32.2,"inr_electricity_per_1000_words":0.0064,"inr_per_1000_tasks":null,"ttft_s":0.14,"output_tps":52.3,"items":117}}},{"id":"ambari-7b","slug":"ambari-7b","name":"Ambari 7B Instruct v0.1","maker":"CognitiveLab","origin":"indian","hosting":"self-hosted","backend":"ollama","params_b":7,"active_params_b":null,"license":"Llama 2 Community","languages":["kn"],"conflict":false,"version":{"ollama_tag":"hf.co/tensorblock/Ambari-7B-Instruct-v0.1-sharded-GGUF:Q3_K_M","digest":"635a035427409fd8ee018f5642dc89e5b5e61b040acbccdc896750210aca93c2","quantization":"Q3_K_M","parameter_size":"6.88B","family":"llama","format":"gguf","source":"Cognitive-Lab/Ambari-7B-Instruct-v0.1","runtime":"ollama 0.40.10","num_ctx":4096,"think":null,"min_max_tokens":null},"weights_gb":3.4,"peak_memory_gb":4.4,"price_usd_per_m":null,"overall":{"product":44.3,"knowledge":11.2,"ttft_s":0.4,"output_tps":24.1},"english":{"tokens_per_word":1.55},"errors":0,"by_lang":{"kn":{"product":44.3,"knowledge":11.2,"tasks":{"rag_qa":{"score":71.4,"n":7,"ci95":[35.9,91.8]},"tool_call":{"score":0,"n":4,"ci95":[0,49]},"intent":{"score":25,"n":4,"ci95":[4.6,69.9]},"support_reply":{"score":50,"n":1,"ci95":[5.5,94.5]},"summary":{"score":75,"n":1,"ci95":[11.8,98.5]},"mmlu_indic":{"score":14,"n":100,"ci95":[8.5,22.1]},"indic_mmlu_pro":{"score":8.3,"n":60,"ci95":[3.6,18.1]}},"fidelity":88.9,"tokens_per_word":2.25,"vs_english":1.13,"inr_per_1000_words":null,"seconds_per_1000_words":93.3,"inr_electricity_per_1000_words":0.0187,"inr_per_1000_tasks":null,"ttft_s":0.4,"output_tps":24.1,"items":177}}},{"id":"qwen3-30b-a3b","slug":"qwen3-30b-a3b","name":"Qwen3 30B A3B","maker":"Alibaba Qwen","origin":"global","hosting":"self-hosted","backend":"ollama","params_b":30,"active_params_b":3,"license":"Apache-2.0","languages":null,"conflict":false,"version":{"ollama_tag":"qwen3:30b","digest":"ad815644918f0eaab341c12b67837cc6dd4562342cdaf118f83d5d554cb37226","quantization":"Q4_K_M","parameter_size":"30.5B","family":"qwen3moe","format":"gguf","source":"Qwen/Qwen3-30B-A3B","runtime":"ollama 0.40.10","num_ctx":4096,"think":false,"min_max_tokens":null},"weights_gb":18.6,"peak_memory_gb":18.7,"price_usd_per_m":null,"overall":{"product":29.4,"knowledge":0.2,"ttft_s":0.33,"output_tps":55.4},"english":{"tokens_per_word":1.5},"errors":0,"by_lang":{"hi":{"product":27.1,"knowledge":0,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":0,"n":4,"ci95":[0,49]},"intent":{"score":0,"n":4,"ci95":[0,49]},"support_reply":{"score":25,"n":1,"ci95":[1.5,88.2]},"summary":{"score":25,"n":1,"ci95":[1.5,88.2]},"mmlu_indic":{"score":0,"n":100,"ci95":[0,3.7]},"indic_mmlu_pro":{"score":0,"n":60,"ci95":[0,6]}},"fidelity":0,"tokens_per_word":4.5,"vs_english":3.43,"inr_per_1000_words":null,"seconds_per_1000_words":78.1,"inr_electricity_per_1000_words":0.0156,"inr_per_1000_tasks":null,"ttft_s":0.37,"output_tps":57.6,"items":177},"kn":{"product":24.3,"knowledge":0,"tasks":{"rag_qa":{"score":71.4,"n":7,"ci95":[35.9,91.8]},"tool_call":{"score":0,"n":4,"ci95":[0,49]},"intent":{"score":0,"n":4,"ci95":[0,49]},"support_reply":{"score":25,"n":1,"ci95":[1.5,88.2]},"summary":{"score":25,"n":1,"ci95":[1.5,88.2]},"mmlu_indic":{"score":0,"n":100,"ci95":[0,3.7]},"indic_mmlu_pro":{"score":0,"n":60,"ci95":[0,6]}},"fidelity":0,"tokens_per_word":10.18,"vs_english":5.29,"inr_per_1000_words":null,"seconds_per_1000_words":183.4,"inr_electricity_per_1000_words":0.0367,"inr_per_1000_tasks":null,"ttft_s":0.5,"output_tps":55.5,"items":177},"ta":{"product":27.1,"knowledge":0,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":0,"n":4,"ci95":[0,49]},"intent":{"score":0,"n":4,"ci95":[0,49]},"support_reply":{"score":25,"n":1,"ci95":[1.5,88.2]},"summary":{"score":25,"n":1,"ci95":[1.5,88.2]},"mmlu_indic":{"score":0,"n":100,"ci95":[0,3.7]},"indic_mmlu_pro":{"score":0,"n":60,"ci95":[0,6]}},"fidelity":0,"tokens_per_word":8.9,"vs_english":4.84,"inr_per_1000_words":null,"seconds_per_1000_words":161.4,"inr_electricity_per_1000_words":0.0323,"inr_per_1000_tasks":null,"ttft_s":0.49,"output_tps":55.2,"items":177},"te":{"product":27.1,"knowledge":0.5,"tasks":{"rag_qa":{"score":85.7,"n":7,"ci95":[48.7,97.4]},"tool_call":{"score":0,"n":4,"ci95":[0,49]},"intent":{"score":0,"n":4,"ci95":[0,49]},"support_reply":{"score":25,"n":1,"ci95":[1.5,88.2]},"summary":{"score":25,"n":1,"ci95":[1.5,88.2]},"mmlu_indic":{"score":1,"n":100,"ci95":[0.2,5.4]},"indic_mmlu_pro":{"score":0,"n":60,"ci95":[0,6]}},"fidelity":0,"tokens_per_word":10.37,"vs_english":5.57,"inr_per_1000_words":null,"seconds_per_1000_words":188.3,"inr_electricity_per_1000_words":0.0377,"inr_per_1000_tasks":null,"ttft_s":0.56,"output_tps":55.1,"items":177},"ml":{"product":31.8,"knowledge":0,"tasks":{"rag_qa":{"score":71.4,"n":7,"ci95":[35.9,91.8]},"tool_call":{"score":0,"n":4,"ci95":[0,49]},"intent":{"score":0,"n":4,"ci95":[0,49]},"support_reply":{"score":25,"n":1,"ci95":[1.5,88.2]},"summary":{"score":62.5,"n":1,"ci95":[8.3,96.8]},"mmlu_indic":{"score":0,"n":100,"ci95":[0,3.7]}},"fidelity":0,"tokens_per_word":10.47,"vs_english":5.44,"inr_per_1000_words":null,"seconds_per_1000_words":195.6,"inr_electricity_per_1000_words":0.0391,"inr_per_1000_tasks":null,"ttft_s":0.53,"output_tps":53.5,"items":117},"hinglish":{"product":39.1,"knowledge":1,"tasks":{"rag_qa":{"score":14.3,"n":7,"ci95":[2.6,51.3]},"tool_call":{"score":0,"n":4,"ci95":[0,49]},"intent":{"score":0,"n":4,"ci95":[0,49]},"support_reply":{"score":81.2,"n":1,"ci95":[13.8,99.2]},"summary":{"score":100,"n":1,"ci95":[20.7,100]},"mmlu_indic":{"score":1,"n":100,"ci95":[0.2,5.4]}},"fidelity":100,"tokens_per_word":1.75,"vs_english":1.25,"inr_per_1000_words":null,"seconds_per_1000_words":32.3,"inr_electricity_per_1000_words":0.0065,"inr_per_1000_tasks":null,"ttft_s":0.24,"output_tps":54.1,"items":117}}},{"id":"kalike-d20","slug":"kalike-d20","name":"Kalike d20","maker":"Suhas Bhairav","origin":"indian","hosting":"self-hosted","backend":"kalike","params_b":0.56,"active_params_b":null,"license":"MIT","languages":["kn"],"conflict":true,"version":{"source_repo":"SuhasBhairav/kalike","checkpoint":"d20/sft/meta_000068.json","step":68,"quantization":"fp32 (MPS)","runtime":"nanochat (torch)"},"weights_gb":1.8,"peak_memory_gb":4.3,"price_usd_per_m":null,"overall":{"product":19.1,"knowledge":0,"ttft_s":0.1,"output_tps":56.3},"english":{"tokens_per_word":5.53},"errors":0,"by_lang":{"kn":{"product":19.1,"knowledge":0,"tasks":{"rag_qa":{"score":14.3,"n":7,"ci95":[2.6,51.3]},"tool_call":{"score":0,"n":4,"ci95":[0,49]},"intent":{"score":0,"n":4,"ci95":[0,49]},"support_reply":{"score":50,"n":1,"ci95":[5.5,94.5]},"summary":{"score":31.2,"n":1,"ci95":[2.2,90]},"mmlu_indic":{"score":0,"n":100,"ci95":[0,3.7]},"indic_mmlu_pro":{"score":0,"n":60,"ci95":[0,6]}},"fidelity":100,"tokens_per_word":1.9,"vs_english":0.27,"inr_per_1000_words":null,"seconds_per_1000_words":33.7,"inr_electricity_per_1000_words":0.0067,"inr_per_1000_tasks":null,"ttft_s":0.1,"output_tps":56.3,"items":177}}}]}