{ "generated": "2026-10-26T18:49:10.589Z", "settings": { "temperature": 0.7, "top_p": 0.9, "num_predict": 350, "runs": 3, "num_ctx": 4096 }, "counts": { "trials": 1438, "prompts": 32, "constraints": 52, "models": 15 }, "ranking": [ "llama3.2-3b", "gemma2-2b", "granite3.3-2b", "qwen2.5-7b", "qwen3-1.7b", "falcon3-3b", "gemma3-1b", "llama3.2-1b", "qwen2.5-3b", "smollm2-1.7b", "qwen3-0.6b", "phi3.5-mini", "qwen2.5-1.5b", "qwen2.5-0.5b", "tinyllama-1.1b" ], "models": { "qwen2.5-0.5b": { "id": "qwen2.5-0.5b", "label": "Qwen2.5 0.5B Instruct", "family": "Qwen", "params": "0.5B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 33, "n": 96, "rate": 0.344, "wilson": [ 0.256, 0.443 ], "cluster_ci": [ 0.208, 0.49 ] }, "loose": { "k": 83, "n": 156, "rate": 0.532, "wilson": [ 0.454, 0.609 ] }, "by_type": { "exact_sentences": { "k": 9, "n": 21 }, "max_words": { "k": 9, "n": 9 }, "forbid_words": { "k": 6, "n": 12 }, "exact_bullets": { "k": 5, "n": 6 }, "min_words": { "k": 8, "n": 9 }, "include_words": { "k": 6, "n": 9 }, "all_lowercase": { "k": 0, "n": 6 }, "all_uppercase": { "k": 2, "n": 6 }, "no_commas": { "k": 3, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 2, "n": 6 }, "postscript": { "k": 1, "n": 6 }, "exact_paragraphs": { "k": 0, "n": 3 }, "placeholders_at_least": { "k": 3, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 3, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 0, "n": 3 }, "wrap_quotes": { "k": 3, "n": 6 }, "bold_sections": { "k": 1, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 2, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 2, "n": 3 } }, "per_prompt": { "i01": { "k": 0, "n": 3 }, "i02": { "k": 0, "n": 3 }, "i03": { "k": 2, "n": 3 }, "i04": { "k": 1, "n": 3 }, "i05": { "k": 0, "n": 3 }, "i06": { "k": 0, "n": 3 }, "i07": { "k": 1, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 1, "n": 3 }, "i10": { "k": 1, "n": 3 }, "i11": { "k": 0, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 0, "n": 3 }, "i17": { "k": 0, "n": 3 }, "i18": { "k": 1, "n": 3 }, "i19": { "k": 1, "n": 3 }, "i20": { "k": 0, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 2, "n": 3 }, "i23": { "k": 0, "n": 3 }, "i24": { "k": 0, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 3, "n": 3 }, "i27": { "k": 0, "n": 3 }, "i28": { "k": 0, "n": 3 }, "i29": { "k": 0, "n": 3 }, "i30": { "k": 0, "n": 3 }, "i31": { "k": 0, "n": 3 }, "i32": { "k": 2, "n": 3 } }, "runs_consistent_prompts": 23, "median_tok_per_s": 58.1, "median_wall_s": 0.82, "median_tokens": 36, "loadavg": { "min": 7.5, "max": 13.37, "median": 12.62 } }, "qwen2.5-1.5b": { "id": "qwen2.5-1.5b", "label": "Qwen2.5 1.5B Instruct", "family": "Qwen", "params": "1.5B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 51, "n": 96, "rate": 0.531, "wilson": [ 0.432, 0.628 ], "cluster_ci": [ 0.385, 0.677 ] }, "loose": { "k": 108, "n": 156, "rate": 0.692, "wilson": [ 0.616, 0.759 ] }, "by_type": { "exact_sentences": { "k": 12, "n": 21 }, "max_words": { "k": 7, "n": 9 }, "forbid_words": { "k": 10, "n": 12 }, "exact_bullets": { "k": 6, "n": 6 }, "min_words": { "k": 7, "n": 9 }, "include_words": { "k": 6, "n": 9 }, "all_lowercase": { "k": 6, "n": 6 }, "all_uppercase": { "k": 6, "n": 6 }, "no_commas": { "k": 2, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 4, "n": 6 }, "postscript": { "k": 0, "n": 6 }, "exact_paragraphs": { "k": 0, "n": 3 }, "placeholders_at_least": { "k": 6, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 7, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 0, "n": 3 }, "wrap_quotes": { "k": 4, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 1, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 3, "n": 3 } }, "per_prompt": { "i01": { "k": 2, "n": 3 }, "i02": { "k": 1, "n": 3 }, "i03": { "k": 3, "n": 3 }, "i04": { "k": 0, "n": 3 }, "i05": { "k": 2, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 1, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 1, "n": 3 }, "i10": { "k": 0, "n": 3 }, "i11": { "k": 0, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 0, "n": 3 }, "i17": { "k": 1, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 0, "n": 3 }, "i20": { "k": 1, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 3, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 1, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 3, "n": 3 }, "i27": { "k": 0, "n": 3 }, "i28": { "k": 1, "n": 3 }, "i29": { "k": 0, "n": 3 }, "i30": { "k": 0, "n": 3 }, "i31": { "k": 2, "n": 3 }, "i32": { "k": 2, "n": 3 } }, "runs_consistent_prompts": 21, "median_tok_per_s": 28.2, "median_wall_s": 1.98, "median_tokens": 46.5, "loadavg": { "min": 11.28, "max": 14.55, "median": 12.99 } }, "qwen2.5-3b": { "id": "qwen2.5-3b", "label": "Qwen2.5 3B Instruct", "family": "Qwen", "params": "3B", "quant": "Q4_K_M", "only_if": false, "trials": 95, "strict": { "k": 64, "n": 95, "rate": 0.674, "wilson": [ 0.574, 0.76 ], "cluster_ci": [ 0.521, 0.813 ] }, "loose": { "k": 119, "n": 155, "rate": 0.768, "wilson": [ 0.695, 0.827 ] }, "by_type": { "exact_sentences": { "k": 12, "n": 21 }, "max_words": { "k": 9, "n": 9 }, "forbid_words": { "k": 11, "n": 12 }, "exact_bullets": { "k": 3, "n": 5 }, "min_words": { "k": 3, "n": 9 }, "include_words": { "k": 5, "n": 9 }, "all_lowercase": { "k": 6, "n": 6 }, "all_uppercase": { "k": 6, "n": 6 }, "no_commas": { "k": 6, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 4, "n": 6 }, "postscript": { "k": 6, "n": 6 }, "exact_paragraphs": { "k": 1, "n": 3 }, "placeholders_at_least": { "k": 6, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 9, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 0, "n": 3 }, "wrap_quotes": { "k": 6, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 2, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 3, "n": 3 } }, "per_prompt": { "i01": { "k": 3, "n": 3 }, "i02": { "k": 2, "n": 3 }, "i03": { "k": 3, "n": 3 }, "i04": { "k": 0, "n": 3 }, "i05": { "k": 0, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 3, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 1, "n": 3 }, "i10": { "k": 3, "n": 3 }, "i11": { "k": 1, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 0, "n": 3 }, "i17": { "k": 3, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 0, "n": 3 }, "i20": { "k": 2, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 3, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 3, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 0, "n": 2 }, "i27": { "k": 3, "n": 3 }, "i28": { "k": 3, "n": 3 }, "i29": { "k": 0, "n": 3 }, "i30": { "k": 0, "n": 3 }, "i31": { "k": 3, "n": 3 }, "i32": { "k": 1, "n": 3 } }, "runs_consistent_prompts": 26, "median_tok_per_s": 17.3, "median_wall_s": 3.17, "median_tokens": 35, "loadavg": { "min": 10.54, "max": 18.35, "median": 11.44 } }, "qwen2.5-7b": { "id": "qwen2.5-7b", "label": "Qwen2.5 7B Instruct", "family": "Qwen", "params": "7B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 74, "n": 96, "rate": 0.771, "wilson": [ 0.677, 0.844 ], "cluster_ci": [ 0.646, 0.896 ] }, "loose": { "k": 132, "n": 156, "rate": 0.846, "wilson": [ 0.781, 0.894 ] }, "by_type": { "exact_sentences": { "k": 12, "n": 21 }, "max_words": { "k": 9, "n": 9 }, "forbid_words": { "k": 12, "n": 12 }, "exact_bullets": { "k": 6, "n": 6 }, "min_words": { "k": 6, "n": 9 }, "include_words": { "k": 7, "n": 9 }, "all_lowercase": { "k": 5, "n": 6 }, "all_uppercase": { "k": 6, "n": 6 }, "no_commas": { "k": 9, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 6, "n": 6 }, "postscript": { "k": 6, "n": 6 }, "exact_paragraphs": { "k": 1, "n": 3 }, "placeholders_at_least": { "k": 6, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 9, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 1, "n": 3 }, "wrap_quotes": { "k": 6, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 0, "n": 3 }, "exact_words": { "k": 1, "n": 3 }, "letter_at_least": { "k": 3, "n": 3 } }, "per_prompt": { "i01": { "k": 3, "n": 3 }, "i02": { "k": 3, "n": 3 }, "i03": { "k": 3, "n": 3 }, "i04": { "k": 1, "n": 3 }, "i05": { "k": 0, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 3, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 3, "n": 3 }, "i10": { "k": 3, "n": 3 }, "i11": { "k": 1, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 1, "n": 3 }, "i17": { "k": 3, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 0, "n": 3 }, "i20": { "k": 0, "n": 3 }, "i21": { "k": 1, "n": 3 }, "i22": { "k": 3, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 3, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 3, "n": 3 }, "i27": { "k": 3, "n": 3 }, "i28": { "k": 3, "n": 3 }, "i29": { "k": 1, "n": 3 }, "i30": { "k": 0, "n": 3 }, "i31": { "k": 3, "n": 3 }, "i32": { "k": 3, "n": 3 } }, "runs_consistent_prompts": 27, "median_tok_per_s": 5.3, "median_wall_s": 11.22, "median_tokens": 37.5, "loadavg": { "min": 10.14, "max": 31.82, "median": 13.2 } }, "llama3.2-3b": { "id": "llama3.2-3b", "label": "Llama 3.2 3B Instruct", "family": "Llama", "params": "3B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 79, "n": 96, "rate": 0.823, "wilson": [ 0.735, 0.886 ], "cluster_ci": [ 0.708, 0.927 ] }, "loose": { "k": 139, "n": 156, "rate": 0.891, "wilson": [ 0.832, 0.931 ] }, "by_type": { "exact_sentences": { "k": 17, "n": 21 }, "max_words": { "k": 8, "n": 9 }, "forbid_words": { "k": 12, "n": 12 }, "exact_bullets": { "k": 6, "n": 6 }, "min_words": { "k": 9, "n": 9 }, "include_words": { "k": 9, "n": 9 }, "all_lowercase": { "k": 6, "n": 6 }, "all_uppercase": { "k": 3, "n": 6 }, "no_commas": { "k": 9, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 6, "n": 6 }, "postscript": { "k": 6, "n": 6 }, "exact_paragraphs": { "k": 3, "n": 3 }, "placeholders_at_least": { "k": 4, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 6, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 0, "n": 3 }, "wrap_quotes": { "k": 6, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 3, "n": 3 }, "exact_words": { "k": 2, "n": 3 }, "letter_at_least": { "k": 3, "n": 3 } }, "per_prompt": { "i01": { "k": 0, "n": 3 }, "i02": { "k": 3, "n": 3 }, "i03": { "k": 3, "n": 3 }, "i04": { "k": 3, "n": 3 }, "i05": { "k": 3, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 3, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 3, "n": 3 }, "i10": { "k": 3, "n": 3 }, "i11": { "k": 3, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 0, "n": 3 }, "i17": { "k": 3, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 3, "n": 3 }, "i20": { "k": 3, "n": 3 }, "i21": { "k": 2, "n": 3 }, "i22": { "k": 3, "n": 3 }, "i23": { "k": 1, "n": 3 }, "i24": { "k": 2, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 3, "n": 3 }, "i27": { "k": 3, "n": 3 }, "i28": { "k": 2, "n": 3 }, "i29": { "k": 1, "n": 3 }, "i30": { "k": 0, "n": 3 }, "i31": { "k": 2, "n": 3 }, "i32": { "k": 3, "n": 3 } }, "runs_consistent_prompts": 26, "median_tok_per_s": 19.5, "median_wall_s": 3.9, "median_tokens": 67, "loadavg": { "min": 10.24, "max": 14.42, "median": 11.15 } }, "gemma2-2b": { "id": "gemma2-2b", "label": "Gemma 2 2B Instruct", "family": "Gemma", "params": "2B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 77, "n": 96, "rate": 0.802, "wilson": [ 0.711, 0.869 ], "cluster_ci": [ 0.698, 0.896 ] }, "loose": { "k": 133, "n": 156, "rate": 0.853, "wilson": [ 0.788, 0.9 ] }, "by_type": { "exact_sentences": { "k": 20, "n": 21 }, "max_words": { "k": 9, "n": 9 }, "forbid_words": { "k": 12, "n": 12 }, "exact_bullets": { "k": 5, "n": 6 }, "min_words": { "k": 9, "n": 9 }, "include_words": { "k": 8, "n": 9 }, "all_lowercase": { "k": 6, "n": 6 }, "all_uppercase": { "k": 5, "n": 6 }, "no_commas": { "k": 4, "n": 9 }, "title_brackets": { "k": 5, "n": 6 }, "ends_with": { "k": 6, "n": 6 }, "postscript": { "k": 6, "n": 6 }, "exact_paragraphs": { "k": 3, "n": 3 }, "placeholders_at_least": { "k": 3, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 9, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 1, "n": 3 }, "wrap_quotes": { "k": 5, "n": 6 }, "bold_sections": { "k": 1, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 2, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 2, "n": 3 } }, "per_prompt": { "i01": { "k": 3, "n": 3 }, "i02": { "k": 3, "n": 3 }, "i03": { "k": 3, "n": 3 }, "i04": { "k": 3, "n": 3 }, "i05": { "k": 3, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 3, "n": 3 }, "i08": { "k": 2, "n": 3 }, "i09": { "k": 3, "n": 3 }, "i10": { "k": 3, "n": 3 }, "i11": { "k": 3, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 1, "n": 3 }, "i17": { "k": 2, "n": 3 }, "i18": { "k": 1, "n": 3 }, "i19": { "k": 3, "n": 3 }, "i20": { "k": 2, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 2, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 1, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 2, "n": 3 }, "i27": { "k": 3, "n": 3 }, "i28": { "k": 3, "n": 3 }, "i29": { "k": 0, "n": 3 }, "i30": { "k": 2, "n": 3 }, "i31": { "k": 3, "n": 3 }, "i32": { "k": 2, "n": 3 } }, "runs_consistent_prompts": 22, "median_tok_per_s": 22.2, "median_wall_s": 2.48, "median_tokens": 44, "loadavg": { "min": 7.3, "max": 11.42, "median": 9.99 } }, "phi3.5-mini": { "id": "phi3.5-mini", "label": "Phi-3.5 mini Instruct", "family": "Phi", "params": "3.8B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 63, "n": 96, "rate": 0.656, "wilson": [ 0.557, 0.744 ], "cluster_ci": [ 0.51, 0.792 ] }, "loose": { "k": 119, "n": 156, "rate": 0.763, "wilson": [ 0.69, 0.823 ] }, "by_type": { "exact_sentences": { "k": 17, "n": 21 }, "max_words": { "k": 7, "n": 9 }, "forbid_words": { "k": 12, "n": 12 }, "exact_bullets": { "k": 0, "n": 6 }, "min_words": { "k": 9, "n": 9 }, "include_words": { "k": 6, "n": 9 }, "all_lowercase": { "k": 4, "n": 6 }, "all_uppercase": { "k": 6, "n": 6 }, "no_commas": { "k": 1, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 5, "n": 6 }, "postscript": { "k": 5, "n": 6 }, "placeholders_at_least": { "k": 6, "n": 6 }, "exact_numbered": { "k": 1, "n": 3 }, "json_keys": { "k": 9, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 2, "n": 3 }, "wrap_quotes": { "k": 5, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 2, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 3, "n": 3 }, "exact_paragraphs": { "k": 1, "n": 3 } }, "per_prompt": { "i01": { "k": 0, "n": 3 }, "i02": { "k": 3, "n": 3 }, "i03": { "k": 0, "n": 3 }, "i04": { "k": 2, "n": 3 }, "i05": { "k": 3, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 1, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 3, "n": 3 }, "i10": { "k": 3, "n": 3 }, "i11": { "k": 1, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 1, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 2, "n": 3 }, "i17": { "k": 3, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 2, "n": 3 }, "i20": { "k": 2, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 3, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 0, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 0, "n": 3 }, "i27": { "k": 2, "n": 3 }, "i28": { "k": 3, "n": 3 }, "i29": { "k": 0, "n": 3 }, "i30": { "k": 3, "n": 3 }, "i31": { "k": 2, "n": 3 }, "i32": { "k": 0, "n": 3 } }, "runs_consistent_prompts": 23, "median_tok_per_s": 13.9, "median_wall_s": 8.33, "median_tokens": 90.5, "loadavg": { "min": 7.06, "max": 13.7, "median": 10.785 } }, "smollm2-1.7b": { "id": "smollm2-1.7b", "label": "SmolLM2 1.7B Instruct", "family": "SmolLM", "params": "1.7B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 64, "n": 96, "rate": 0.667, "wilson": [ 0.568, 0.753 ], "cluster_ci": [ 0.521, 0.802 ] }, "loose": { "k": 122, "n": 156, "rate": 0.782, "wilson": [ 0.711, 0.84 ] }, "by_type": { "exact_sentences": { "k": 11, "n": 21 }, "max_words": { "k": 9, "n": 9 }, "forbid_words": { "k": 8, "n": 12 }, "exact_bullets": { "k": 6, "n": 6 }, "min_words": { "k": 9, "n": 9 }, "include_words": { "k": 8, "n": 9 }, "all_lowercase": { "k": 4, "n": 6 }, "all_uppercase": { "k": 6, "n": 6 }, "no_commas": { "k": 3, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 4, "n": 6 }, "postscript": { "k": 6, "n": 6 }, "exact_paragraphs": { "k": 3, "n": 3 }, "placeholders_at_least": { "k": 4, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 9, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 1, "n": 3 }, "wrap_quotes": { "k": 5, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 2, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 3, "n": 3 } }, "per_prompt": { "i01": { "k": 0, "n": 3 }, "i02": { "k": 0, "n": 3 }, "i03": { "k": 3, "n": 3 }, "i04": { "k": 3, "n": 3 }, "i05": { "k": 0, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 0, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 1, "n": 3 }, "i10": { "k": 3, "n": 3 }, "i11": { "k": 3, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 1, "n": 3 }, "i17": { "k": 2, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 1, "n": 3 }, "i20": { "k": 2, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 3, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 1, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 3, "n": 3 }, "i27": { "k": 3, "n": 3 }, "i28": { "k": 3, "n": 3 }, "i29": { "k": 0, "n": 3 }, "i30": { "k": 1, "n": 3 }, "i31": { "k": 2, "n": 3 }, "i32": { "k": 2, "n": 3 } }, "runs_consistent_prompts": 23, "median_tok_per_s": 36.8, "median_wall_s": 1.77, "median_tokens": 51.5, "loadavg": { "min": 9.94, "max": 11.26, "median": 10.75 } }, "llama3.2-1b": { "id": "llama3.2-1b", "label": "Llama 3.2 1B Instruct", "family": "Llama", "params": "1B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 65, "n": 96, "rate": 0.677, "wilson": [ 0.578, 0.762 ], "cluster_ci": [ 0.542, 0.813 ] }, "loose": { "k": 124, "n": 156, "rate": 0.795, "wilson": [ 0.725, 0.851 ] }, "by_type": { "exact_sentences": { "k": 16, "n": 21 }, "max_words": { "k": 7, "n": 9 }, "forbid_words": { "k": 12, "n": 12 }, "exact_bullets": { "k": 6, "n": 6 }, "min_words": { "k": 9, "n": 9 }, "include_words": { "k": 6, "n": 9 }, "all_lowercase": { "k": 6, "n": 6 }, "all_uppercase": { "k": 6, "n": 6 }, "no_commas": { "k": 9, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 1, "n": 6 }, "postscript": { "k": 3, "n": 6 }, "exact_paragraphs": { "k": 2, "n": 3 }, "placeholders_at_least": { "k": 3, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 7, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 0, "n": 3 }, "wrap_quotes": { "k": 5, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 3, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 2, "n": 3 } }, "per_prompt": { "i01": { "k": 0, "n": 3 }, "i02": { "k": 3, "n": 3 }, "i03": { "k": 3, "n": 3 }, "i04": { "k": 1, "n": 3 }, "i05": { "k": 3, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 2, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 0, "n": 3 }, "i10": { "k": 1, "n": 3 }, "i11": { "k": 2, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 0, "n": 3 }, "i17": { "k": 2, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 3, "n": 3 }, "i20": { "k": 3, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 2, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 2, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 3, "n": 3 }, "i27": { "k": 2, "n": 3 }, "i28": { "k": 1, "n": 3 }, "i29": { "k": 0, "n": 3 }, "i30": { "k": 3, "n": 3 }, "i31": { "k": 0, "n": 3 }, "i32": { "k": 2, "n": 3 } }, "runs_consistent_prompts": 22, "median_tok_per_s": 39, "median_wall_s": 1.75, "median_tokens": 62.5, "loadavg": { "min": 9.68, "max": 12.54, "median": 10.9 } }, "gemma3-1b": { "id": "gemma3-1b", "label": "Gemma 3 1B Instruct", "family": "Gemma", "params": "1B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 68, "n": 96, "rate": 0.708, "wilson": [ 0.611, 0.79 ], "cluster_ci": [ 0.552, 0.844 ] }, "loose": { "k": 123, "n": 156, "rate": 0.788, "wilson": [ 0.718, 0.845 ] }, "by_type": { "exact_sentences": { "k": 17, "n": 21 }, "max_words": { "k": 9, "n": 9 }, "forbid_words": { "k": 12, "n": 12 }, "exact_bullets": { "k": 0, "n": 6 }, "min_words": { "k": 9, "n": 9 }, "include_words": { "k": 6, "n": 9 }, "all_lowercase": { "k": 6, "n": 6 }, "all_uppercase": { "k": 3, "n": 6 }, "no_commas": { "k": 7, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 6, "n": 6 }, "postscript": { "k": 3, "n": 6 }, "exact_paragraphs": { "k": 3, "n": 3 }, "placeholders_at_least": { "k": 6, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 9, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 0, "n": 3 }, "wrap_quotes": { "k": 0, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 3, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 3, "n": 3 } }, "per_prompt": { "i01": { "k": 3, "n": 3 }, "i02": { "k": 3, "n": 3 }, "i03": { "k": 0, "n": 3 }, "i04": { "k": 2, "n": 3 }, "i05": { "k": 3, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 3, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 3, "n": 3 }, "i10": { "k": 0, "n": 3 }, "i11": { "k": 3, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 0, "n": 3 }, "i17": { "k": 0, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 3, "n": 3 }, "i20": { "k": 3, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 3, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 3, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 0, "n": 3 }, "i27": { "k": 3, "n": 3 }, "i28": { "k": 3, "n": 3 }, "i29": { "k": 1, "n": 3 }, "i30": { "k": 0, "n": 3 }, "i31": { "k": 2, "n": 3 }, "i32": { "k": 0, "n": 3 } }, "runs_consistent_prompts": 29, "median_tok_per_s": 42.1, "median_wall_s": 1.65, "median_tokens": 58, "loadavg": { "min": 9.32, "max": 11.41, "median": 10.105 } }, "qwen3-0.6b": { "id": "qwen3-0.6b", "label": "Qwen3 0.6B (non-thinking)", "family": "Qwen", "params": "0.6B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 64, "n": 96, "rate": 0.667, "wilson": [ 0.568, 0.753 ], "cluster_ci": [ 0.51, 0.792 ] }, "loose": { "k": 121, "n": 156, "rate": 0.776, "wilson": [ 0.704, 0.834 ] }, "by_type": { "exact_sentences": { "k": 8, "n": 21 }, "max_words": { "k": 8, "n": 9 }, "forbid_words": { "k": 12, "n": 12 }, "exact_bullets": { "k": 6, "n": 6 }, "min_words": { "k": 5, "n": 9 }, "include_words": { "k": 9, "n": 9 }, "all_lowercase": { "k": 6, "n": 6 }, "all_uppercase": { "k": 6, "n": 6 }, "no_commas": { "k": 2, "n": 9 }, "title_brackets": { "k": 5, "n": 6 }, "ends_with": { "k": 4, "n": 6 }, "postscript": { "k": 5, "n": 6 }, "exact_paragraphs": { "k": 2, "n": 3 }, "placeholders_at_least": { "k": 6, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 9, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 3, "n": 3 }, "wrap_quotes": { "k": 6, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 1, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 3, "n": 3 } }, "per_prompt": { "i01": { "k": 0, "n": 3 }, "i02": { "k": 3, "n": 3 }, "i03": { "k": 3, "n": 3 }, "i04": { "k": 2, "n": 3 }, "i05": { "k": 0, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 2, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 1, "n": 3 }, "i10": { "k": 2, "n": 3 }, "i11": { "k": 2, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 3, "n": 3 }, "i17": { "k": 2, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 3, "n": 3 }, "i20": { "k": 1, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 3, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 0, "n": 3 }, "i25": { "k": 1, "n": 3 }, "i26": { "k": 3, "n": 3 }, "i27": { "k": 3, "n": 3 }, "i28": { "k": 3, "n": 3 }, "i29": { "k": 0, "n": 3 }, "i30": { "k": 0, "n": 3 }, "i31": { "k": 0, "n": 3 }, "i32": { "k": 3, "n": 3 } }, "runs_consistent_prompts": 24, "median_tok_per_s": 59.6, "median_wall_s": 0.72, "median_tokens": 42.5, "loadavg": { "min": 13.77, "max": 50.31, "median": 17.62 } }, "qwen3-1.7b": { "id": "qwen3-1.7b", "label": "Qwen3 1.7B (non-thinking)", "family": "Qwen", "params": "1.7B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 74, "n": 96, "rate": 0.771, "wilson": [ 0.677, 0.844 ], "cluster_ci": [ 0.635, 0.885 ] }, "loose": { "k": 133, "n": 156, "rate": 0.853, "wilson": [ 0.788, 0.9 ] }, "by_type": { "exact_sentences": { "k": 12, "n": 21 }, "max_words": { "k": 9, "n": 9 }, "forbid_words": { "k": 8, "n": 12 }, "exact_bullets": { "k": 6, "n": 6 }, "min_words": { "k": 6, "n": 9 }, "include_words": { "k": 8, "n": 9 }, "all_lowercase": { "k": 6, "n": 6 }, "all_uppercase": { "k": 6, "n": 6 }, "no_commas": { "k": 7, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 6, "n": 6 }, "postscript": { "k": 6, "n": 6 }, "exact_paragraphs": { "k": 3, "n": 3 }, "placeholders_at_least": { "k": 6, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 9, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 3, "n": 3 }, "wrap_quotes": { "k": 6, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 2, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 3, "n": 3 } }, "per_prompt": { "i01": { "k": 3, "n": 3 }, "i02": { "k": 0, "n": 3 }, "i03": { "k": 3, "n": 3 }, "i04": { "k": 0, "n": 3 }, "i05": { "k": 0, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 2, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 3, "n": 3 }, "i10": { "k": 3, "n": 3 }, "i11": { "k": 3, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 3, "n": 3 }, "i17": { "k": 3, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 1, "n": 3 }, "i20": { "k": 2, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 3, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 2, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 3, "n": 3 }, "i27": { "k": 3, "n": 3 }, "i28": { "k": 3, "n": 3 }, "i29": { "k": 2, "n": 3 }, "i30": { "k": 2, "n": 3 }, "i31": { "k": 0, "n": 3 }, "i32": { "k": 3, "n": 3 } }, "runs_consistent_prompts": 26, "median_tok_per_s": 32.5, "median_wall_s": 1.59, "median_tokens": 44.5, "loadavg": { "min": 9.59, "max": 10.78, "median": 9.96 } }, "granite3.3-2b": { "id": "granite3.3-2b", "label": "Granite 3.3 2B Instruct", "family": "Granite", "params": "2B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 76, "n": 96, "rate": 0.792, "wilson": [ 0.7, 0.861 ], "cluster_ci": [ 0.667, 0.906 ] }, "loose": { "k": 135, "n": 156, "rate": 0.865, "wilson": [ 0.803, 0.91 ] }, "by_type": { "exact_sentences": { "k": 21, "n": 21 }, "max_words": { "k": 9, "n": 9 }, "forbid_words": { "k": 10, "n": 12 }, "exact_bullets": { "k": 6, "n": 6 }, "min_words": { "k": 9, "n": 9 }, "include_words": { "k": 7, "n": 9 }, "all_lowercase": { "k": 4, "n": 6 }, "all_uppercase": { "k": 5, "n": 6 }, "no_commas": { "k": 6, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 2, "n": 6 }, "postscript": { "k": 6, "n": 6 }, "exact_paragraphs": { "k": 3, "n": 3 }, "placeholders_at_least": { "k": 6, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 8, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 3, "n": 3 }, "wrap_quotes": { "k": 6, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 0, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 3, "n": 3 } }, "per_prompt": { "i01": { "k": 3, "n": 3 }, "i02": { "k": 1, "n": 3 }, "i03": { "k": 3, "n": 3 }, "i04": { "k": 3, "n": 3 }, "i05": { "k": 3, "n": 3 }, "i06": { "k": 3, "n": 3 }, "i07": { "k": 3, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 1, "n": 3 }, "i10": { "k": 3, "n": 3 }, "i11": { "k": 3, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 3, "n": 3 }, "i17": { "k": 3, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 3, "n": 3 }, "i20": { "k": 0, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 3, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 0, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 3, "n": 3 }, "i27": { "k": 3, "n": 3 }, "i28": { "k": 2, "n": 3 }, "i29": { "k": 2, "n": 3 }, "i30": { "k": 2, "n": 3 }, "i31": { "k": 1, "n": 3 }, "i32": { "k": 1, "n": 3 } }, "runs_consistent_prompts": 25, "median_tok_per_s": 14.2, "median_wall_s": 7.08, "median_tokens": 76, "loadavg": { "min": 11.46, "max": 17.15, "median": 13 } }, "tinyllama-1.1b": { "id": "tinyllama-1.1b", "label": "TinyLlama 1.1B Chat", "family": "TinyLlama", "params": "1.1B", "quant": "Q4_K_M", "only_if": false, "trials": 95, "strict": { "k": 15, "n": 95, "rate": 0.158, "wilson": [ 0.098, 0.244 ], "cluster_ci": [ 0.068, 0.276 ] }, "loose": { "k": 43, "n": 155, "rate": 0.277, "wilson": [ 0.213, 0.353 ] }, "by_type": { "exact_sentences": { "k": 4, "n": 21 }, "max_words": { "k": 3, "n": 9 }, "forbid_words": { "k": 1, "n": 12 }, "exact_bullets": { "k": 0, "n": 6 }, "min_words": { "k": 9, "n": 9 }, "include_words": { "k": 7, "n": 9 }, "all_lowercase": { "k": 0, "n": 6 }, "all_uppercase": { "k": 0, "n": 6 }, "no_commas": { "k": 3, "n": 9 }, "title_brackets": { "k": 0, "n": 6 }, "ends_with": { "k": 0, "n": 6 }, "postscript": { "k": 1, "n": 6 }, "exact_paragraphs": { "k": 0, "n": 3 }, "placeholders_at_least": { "k": 3, "n": 6 }, "exact_numbered": { "k": 0, "n": 3 }, "json_keys": { "k": 3, "n": 9 }, "one_of": { "k": 0, "n": 3 }, "word_at_least": { "k": 2, "n": 3 }, "wrap_quotes": { "k": 4, "n": 6 }, "bold_sections": { "k": 0, "n": 3 }, "starts_with": { "k": 2, "n": 6 }, "between_words": { "k": 0, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 1, "n": 2 } }, "per_prompt": { "i01": { "k": 1, "n": 3 }, "i02": { "k": 0, "n": 3 }, "i03": { "k": 0, "n": 3 }, "i04": { "k": 1, "n": 3 }, "i05": { "k": 0, "n": 3 }, "i06": { "k": 0, "n": 3 }, "i07": { "k": 0, "n": 3 }, "i08": { "k": 0, "n": 3 }, "i09": { "k": 0, "n": 3 }, "i10": { "k": 1, "n": 3 }, "i11": { "k": 0, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 0, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 0, "n": 3 }, "i16": { "k": 2, "n": 3 }, "i17": { "k": 1, "n": 3 }, "i18": { "k": 0, "n": 3 }, "i19": { "k": 0, "n": 3 }, "i20": { "k": 0, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 1, "n": 2 }, "i23": { "k": 0, "n": 3 }, "i24": { "k": 0, "n": 3 }, "i25": { "k": 0, "n": 3 }, "i26": { "k": 0, "n": 3 }, "i27": { "k": 0, "n": 3 }, "i28": { "k": 0, "n": 3 }, "i29": { "k": 0, "n": 3 }, "i30": { "k": 0, "n": 3 }, "i31": { "k": 0, "n": 3 }, "i32": { "k": 2, "n": 3 } }, "runs_consistent_prompts": 25, "median_tok_per_s": 31.9, "median_wall_s": 6.26, "median_tokens": 214, "loadavg": { "min": 9.71, "max": 17.54, "median": 11.13 } }, "falcon3-3b": { "id": "falcon3-3b", "label": "Falcon3 3B Instruct", "family": "Falcon", "params": "3B", "quant": "Q4_K_M", "only_if": false, "trials": 96, "strict": { "k": 72, "n": 96, "rate": 0.75, "wilson": [ 0.655, 0.826 ], "cluster_ci": [ 0.604, 0.875 ] }, "loose": { "k": 128, "n": 156, "rate": 0.821, "wilson": [ 0.753, 0.873 ] }, "by_type": { "exact_sentences": { "k": 13, "n": 21 }, "max_words": { "k": 9, "n": 9 }, "forbid_words": { "k": 11, "n": 12 }, "exact_bullets": { "k": 6, "n": 6 }, "min_words": { "k": 6, "n": 9 }, "include_words": { "k": 8, "n": 9 }, "all_lowercase": { "k": 6, "n": 6 }, "all_uppercase": { "k": 2, "n": 6 }, "no_commas": { "k": 9, "n": 9 }, "title_brackets": { "k": 6, "n": 6 }, "ends_with": { "k": 3, "n": 6 }, "postscript": { "k": 6, "n": 6 }, "exact_paragraphs": { "k": 3, "n": 3 }, "placeholders_at_least": { "k": 5, "n": 6 }, "exact_numbered": { "k": 3, "n": 3 }, "json_keys": { "k": 9, "n": 9 }, "one_of": { "k": 3, "n": 3 }, "word_at_least": { "k": 2, "n": 3 }, "wrap_quotes": { "k": 6, "n": 6 }, "bold_sections": { "k": 3, "n": 3 }, "starts_with": { "k": 6, "n": 6 }, "between_words": { "k": 0, "n": 3 }, "exact_words": { "k": 0, "n": 3 }, "letter_at_least": { "k": 3, "n": 3 } }, "per_prompt": { "i01": { "k": 3, "n": 3 }, "i02": { "k": 2, "n": 3 }, "i03": { "k": 3, "n": 3 }, "i04": { "k": 1, "n": 3 }, "i05": { "k": 3, "n": 3 }, "i06": { "k": 0, "n": 3 }, "i07": { "k": 3, "n": 3 }, "i08": { "k": 3, "n": 3 }, "i09": { "k": 3, "n": 3 }, "i10": { "k": 3, "n": 3 }, "i11": { "k": 3, "n": 3 }, "i12": { "k": 3, "n": 3 }, "i13": { "k": 3, "n": 3 }, "i14": { "k": 3, "n": 3 }, "i15": { "k": 3, "n": 3 }, "i16": { "k": 2, "n": 3 }, "i17": { "k": 3, "n": 3 }, "i18": { "k": 3, "n": 3 }, "i19": { "k": 0, "n": 3 }, "i20": { "k": 0, "n": 3 }, "i21": { "k": 0, "n": 3 }, "i22": { "k": 3, "n": 3 }, "i23": { "k": 3, "n": 3 }, "i24": { "k": 3, "n": 3 }, "i25": { "k": 3, "n": 3 }, "i26": { "k": 3, "n": 3 }, "i27": { "k": 3, "n": 3 }, "i28": { "k": 3, "n": 3 }, "i29": { "k": 2, "n": 3 }, "i30": { "k": 0, "n": 3 }, "i31": { "k": 0, "n": 3 }, "i32": { "k": 2, "n": 3 } }, "runs_consistent_prompts": 27, "median_tok_per_s": 14.3, "median_wall_s": 4.95, "median_tokens": 51, "loadavg": { "min": 11.3, "max": 40.4, "median": 12.73 } } }, "hardest": [ { "id": "i21", "k": 3, "n": 45, "rate": 0.067 }, { "id": "i29", "k": 9, "n": 45, "rate": 0.2 }, { "id": "i30", "k": 13, "n": 45, "rate": 0.289 }, { "id": "i16", "k": 18, "n": 45, "rate": 0.4 }, { "id": "i31", "k": 20, "n": 45, "rate": 0.444 }, { "id": "i20", "k": 21, "n": 45, "rate": 0.467 }, { "id": "i24", "k": 21, "n": 45, "rate": 0.467 }, { "id": "i04", "k": 23, "n": 45, "rate": 0.511 } ], "easiest": [ { "id": "i14", "k": 45, "n": 45, "rate": 1 }, { "id": "i12", "k": 45, "n": 45, "rate": 1 }, { "id": "i15", "k": 42, "n": 45, "rate": 0.933 }, { "id": "i08", "k": 41, "n": 45, "rate": 0.911 }, { "id": "i22", "k": 40, "n": 44, "rate": 0.909 } ] }